From a449f9c07c942c03babb6c98245761bb56a6d1a4 Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 20 Aug 2024 13:22:42 +0800 Subject: [PATCH 1/8] add var.dns_prefix_private_cluster --- README.md | 1 + main.tf | 1 + variables.tf | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index 7afda9fb..0cc6af10 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,7 @@ No modules. | [create\_role\_assignments\_for\_application\_gateway](#input\_create\_role\_assignments\_for\_application\_gateway) | (Optional) Whether to create the corresponding role assignments for application gateway or not. Defaults to `true`. | `bool` | `true` | no | | [default\_node\_pool\_fips\_enabled](#input\_default\_node\_pool\_fips\_enabled) | (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. | `bool` | `null` | no | | [disk\_encryption\_set\_id](#input\_disk\_encryption\_set\_id) | (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created. | `string` | `null` | no | +| [dns\_prefix\_private\_cluster](#input\_dns\_prefix\_private\_cluster) | (Optional) Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created. | `string` | `null` | no | | [ebpf\_data\_plane](#input\_ebpf\_data\_plane) | (Optional) Specifies the eBPF data plane used for building the Kubernetes network. Possible value is `cilium`. Changing this forces a new resource to be created. | `string` | `null` | no | | [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Enable node pool autoscaling | `bool` | `false` | no | | [enable\_host\_encryption](#input\_enable\_host\_encryption) | Enable Host Encryption for default node pool. Encryption at host feature must be enabled on the subscription: https://docs.microsoft.com/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli | `bool` | `false` | no | diff --git a/main.tf b/main.tf index c4dc98cc..31737f4b 100644 --- a/main.tf +++ b/main.tf @@ -22,6 +22,7 @@ resource "azurerm_kubernetes_cluster" "main" { azure_policy_enabled = var.azure_policy_enabled disk_encryption_set_id = var.disk_encryption_set_id dns_prefix = var.prefix + dns_prefix_private_cluster = var.dns_prefix_private_cluster image_cleaner_enabled = var.image_cleaner_enabled image_cleaner_interval_hours = var.image_cleaner_interval_hours kubernetes_version = var.kubernetes_version diff --git a/variables.tf b/variables.tf index 5f304b2f..683f0370 100644 --- a/variables.tf +++ b/variables.tf @@ -454,6 +454,12 @@ variable "disk_encryption_set_id" { description = "(Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created." } +variable "dns_prefix_private_cluster" { + type = string + default = null + description = "(Optional) Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created." +} + variable "ebpf_data_plane" { type = string default = null From 7cf3feb7f339210fd884f91f4a330d0102b845df Mon Sep 17 00:00:00 2001 From: hezijie Date: Wed, 21 Aug 2024 09:51:18 +0800 Subject: [PATCH 2/8] remove nullable=false from var.prefix --- README.md | 4 ++-- locals.tf | 2 +- log_analytics.tf | 2 +- variables.tf | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 0cc6af10..983121c1 100644 --- a/README.md +++ b/README.md @@ -319,7 +319,7 @@ No modules. | [create\_role\_assignments\_for\_application\_gateway](#input\_create\_role\_assignments\_for\_application\_gateway) | (Optional) Whether to create the corresponding role assignments for application gateway or not. Defaults to `true`. | `bool` | `true` | no | | [default\_node\_pool\_fips\_enabled](#input\_default\_node\_pool\_fips\_enabled) | (Optional) Should the nodes in this Node Pool have Federal Information Processing Standard enabled? Changing this forces a new resource to be created. | `bool` | `null` | no | | [disk\_encryption\_set\_id](#input\_disk\_encryption\_set\_id) | (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created. | `string` | `null` | no | -| [dns\_prefix\_private\_cluster](#input\_dns\_prefix\_private\_cluster) | (Optional) Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created. | `string` | `null` | no | +| [dns\_prefix\_private\_cluster](#input\_dns\_prefix\_private\_cluster) | (Optional) Specifies the DNS prefix to use with private clusters. Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified. Changing this forces a new resource to be created. | `string` | `null` | no | | [ebpf\_data\_plane](#input\_ebpf\_data\_plane) | (Optional) Specifies the eBPF data plane used for building the Kubernetes network. Possible value is `cilium`. Changing this forces a new resource to be created. | `string` | `null` | no | | [enable\_auto\_scaling](#input\_enable\_auto\_scaling) | Enable node pool autoscaling | `bool` | `false` | no | | [enable\_host\_encryption](#input\_enable\_host\_encryption) | Enable Host Encryption for default node pool. Encryption at host feature must be enabled on the subscription: https://docs.microsoft.com/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli | `bool` | `false` | no | @@ -387,7 +387,7 @@ No modules. | [os\_disk\_type](#input\_os\_disk\_type) | The type of disk which should be used for the Operating System. Possible values are `Ephemeral` and `Managed`. Defaults to `Managed`. Changing this forces a new resource to be created. | `string` | `"Managed"` | no | | [os\_sku](#input\_os\_sku) | (Optional) Specifies the OS SKU used by the agent pool. Possible values include: `Ubuntu`, `CBLMariner`, `Mariner`, `Windows2019`, `Windows2022`. If not specified, the default is `Ubuntu` if OSType=Linux or `Windows2019` if OSType=Windows. And the default Windows OSSKU will be changed to `Windows2022` after Windows2019 is deprecated. Changing this forces a new resource to be created. | `string` | `null` | no | | [pod\_subnet\_id](#input\_pod\_subnet\_id) | (Optional) The ID of the Subnet where the pods in the default Node Pool should exist. Changing this forces a new resource to be created. | `string` | `null` | no | -| [prefix](#input\_prefix) | (Optional) The prefix for the resources created in the specified Azure Resource Group. Omitting this variable requires both `var.cluster_log_analytics_workspace_name` and `var.cluster_name` have been set. | `string` | `""` | no | +| [prefix](#input\_prefix) | (Optional) The prefix for the resources created in the specified Azure Resource Group. Omitting this variable requires both `var.cluster_log_analytics_workspace_name` and `var.cluster_name` have been set. Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified. | `string` | `""` | no | | [private\_cluster\_enabled](#input\_private\_cluster\_enabled) | If true cluster API server will be exposed only on internal IP address and available only in cluster vnet. | `bool` | `false` | no | | [private\_cluster\_public\_fqdn\_enabled](#input\_private\_cluster\_public\_fqdn\_enabled) | (Optional) Specifies whether a Public FQDN for this Private Cluster should be added. Defaults to `false`. | `bool` | `false` | no | | [private\_dns\_zone\_id](#input\_private\_dns\_zone\_id) | (Optional) Either the ID of Private DNS Zone which should be delegated to this Cluster, `System` to have AKS manage this or `None`. In case of `None` you will need to bring your own DNS server and set up resolving, otherwise cluster will have issues after provisioning. Changing this forces a new resource to be created. | `string` | `null` | no | diff --git a/locals.tf b/locals.tf index 2f4d1c3f..1693f408 100644 --- a/locals.tf +++ b/locals.tf @@ -9,7 +9,7 @@ locals { (contains(["patch"], var.automatic_channel_upgrade) && can(regex("^[0-9]{1,}\\.[0-9]{1,}$", var.kubernetes_version)) && (can(regex("^[0-9]{1,}\\.[0-9]{1,}$", var.orchestrator_version)) || var.orchestrator_version == null)) || (contains(["rapid", "stable", "node-image"], var.automatic_channel_upgrade) && var.kubernetes_version == null && var.orchestrator_version == null) ) - cluster_name = coalesce(var.cluster_name, trim("${var.prefix}-aks", "-")) + cluster_name = try(coalesce(var.cluster_name, trim("${var.prefix}-aks", "-")), "aks") # Abstract the decision whether to create an Analytics Workspace or not. create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null diff --git a/log_analytics.tf b/log_analytics.tf index b791ecdd..e85036f9 100644 --- a/log_analytics.tf +++ b/log_analytics.tf @@ -2,7 +2,7 @@ resource "azurerm_log_analytics_workspace" "main" { count = local.create_analytics_workspace ? 1 : 0 location = coalesce(var.location, data.azurerm_resource_group.main.location) - name = coalesce(var.cluster_log_analytics_workspace_name, trim("${var.prefix}-workspace", "-")) + name = try(coalesce(var.cluster_log_analytics_workspace_name, trim("${var.prefix}-workspace", "-")), "aks-workspace") resource_group_name = coalesce(var.log_analytics_workspace_resource_group_name, var.resource_group_name) allow_resource_only_permissions = var.log_analytics_workspace_allow_resource_only_permissions cmk_for_query_forced = var.log_analytics_workspace_cmk_for_query_forced diff --git a/variables.tf b/variables.tf index 683f0370..519d7491 100644 --- a/variables.tf +++ b/variables.tf @@ -457,7 +457,7 @@ variable "disk_encryption_set_id" { variable "dns_prefix_private_cluster" { type = string default = null - description = "(Optional) Specifies the DNS prefix to use with private clusters. Changing this forces a new resource to be created." + description = "(Optional) Specifies the DNS prefix to use with private clusters. Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified. Changing this forces a new resource to be created." } variable "ebpf_data_plane" { @@ -1223,8 +1223,7 @@ variable "pod_subnet_id" { variable "prefix" { type = string default = "" - description = "(Optional) The prefix for the resources created in the specified Azure Resource Group. Omitting this variable requires both `var.cluster_log_analytics_workspace_name` and `var.cluster_name` have been set." - nullable = false + description = "(Optional) The prefix for the resources created in the specified Azure Resource Group. Omitting this variable requires both `var.cluster_log_analytics_workspace_name` and `var.cluster_name` have been set. Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified." } variable "private_cluster_enabled" { From 1118b4e3e2501c54a00d357ddcaed084e39ab415 Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Mon, 9 Sep 2024 10:06:18 +0200 Subject: [PATCH 3/8] Add precondition: Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified --- main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/main.tf b/main.tf index 31737f4b..9f54b1ae 100644 --- a/main.tf +++ b/main.tf @@ -624,6 +624,10 @@ resource "azurerm_kubernetes_cluster" "main" { condition = var.brown_field_application_gateway_for_ingress == null || var.green_field_application_gateway_for_ingress == null error_message = "Either one of `var.brown_field_application_gateway_for_ingress` or `var.green_field_application_gateway_for_ingress` must be `null`." } + precondition { + condition = var.prefix == null || var.dns_prefix_private_cluster == null + error_message = "Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified." + } } } From e344c91bff2939da01fb3845d7c66d6e18282f32 Mon Sep 17 00:00:00 2001 From: Saverio Proto Date: Mon, 9 Sep 2024 10:43:54 +0200 Subject: [PATCH 4/8] Adding more preconditions for the var.dns_prefix_private_cluster scenario --- main.tf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index 9f54b1ae..aad3cb9b 100644 --- a/main.tf +++ b/main.tf @@ -604,8 +604,8 @@ resource "azurerm_kubernetes_cluster" "main" { error_message = "When ebpf_data_plane is set to cilium, one of either network_plugin_mode = `overlay` or pod_subnet_id must be specified." } precondition { - condition = can(coalesce(var.cluster_name, var.prefix)) - error_message = "You must set one of `var.cluster_name` and `var.prefix` to create `azurerm_kubernetes_cluster.main`." + condition = can(coalesce(var.cluster_name, var.prefix, var.dns_prefix_private_cluster)) + error_message = "You must set one of `var.cluster_name`,`var.prefix`,`var.dns_prefix_private_cluster` to create `azurerm_kubernetes_cluster.main`." } precondition { condition = var.automatic_channel_upgrade != "node-image" || var.node_os_channel_upgrade == "NodeImage" @@ -628,6 +628,14 @@ resource "azurerm_kubernetes_cluster" "main" { condition = var.prefix == null || var.dns_prefix_private_cluster == null error_message = "Only one of `var.prefix,var.dns_prefix_private_cluster` can be specified." } + precondition { + condition = var.dns_prefix_private_cluster == null || var.private_cluster_enabled + error_message = "When `dns_prefix_private_cluster` is set, `private_cluster_enabled` must be set to `true`." + } + precondition { + condition = var.dns_prefix_private_cluster != null || var.identity_type == "UserAssigned" || var.client_id != "" + error_message = "A user assigned identity or a service principal must be used when using a custom private dns zone" + } } } From 02d0df6ca6b78c591421440ea6498f9cdd66ae87 Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 10:54:00 +0800 Subject: [PATCH 5/8] add private dns zone name check --- locals.tf | 6 ++++++ main.tf | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/locals.tf b/locals.tf index 1693f408..08f55ed0 100644 --- a/locals.tf +++ b/locals.tf @@ -58,4 +58,10 @@ locals { subnet_ids = toset([for id in local.potential_subnet_ids : id if id != null]) use_brown_field_gw_for_ingress = var.brown_field_application_gateway_for_ingress != null use_green_field_gw_for_ingress = var.green_field_application_gateway_for_ingress != null + valid_private_dns_zone_regexs = [ + "private\\.[a-z]+\\.azmk8s\\.io", + "privatelink\\.[a-z]+\\.azmk8s\\.io", + "[a-zA-Z0-9\\-]{1,32}\\.private\\.[a-z]+\\.azmk8s\\.io", + "[a-zA-Z0-9\\-]{1,32}\\.privatelink\\.[a-z]+\\.azmk8s\\.io", + ] } diff --git a/main.tf b/main.tf index aad3cb9b..805ee9af 100644 --- a/main.tf +++ b/main.tf @@ -636,6 +636,10 @@ resource "azurerm_kubernetes_cluster" "main" { condition = var.dns_prefix_private_cluster != null || var.identity_type == "UserAssigned" || var.client_id != "" error_message = "A user assigned identity or a service principal must be used when using a custom private dns zone" } + precondition { + condition = var.private_dns_zone_id == null ? true : (anytrue([for r in local.valid_private_dns_zone_regexs : try(regex(r, reverse(split("/", var.private_dns_zone_id))[0]) == reverse(split("/", var.private_dns_zone_id))[0], false)])) + error_message = "Private DNS zone must be in one of the following format: `privatelink..azmk8s.io`, `.privatelink..azmk8s.io`, `private..azmk8s.io`, `.private..azmk8s.io`" + } } } From 6c537edc2544642e5ddef9a17b03333779e091d3 Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 10:54:42 +0800 Subject: [PATCH 6/8] adjust error message --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 805ee9af..1570997c 100644 --- a/main.tf +++ b/main.tf @@ -638,7 +638,7 @@ resource "azurerm_kubernetes_cluster" "main" { } precondition { condition = var.private_dns_zone_id == null ? true : (anytrue([for r in local.valid_private_dns_zone_regexs : try(regex(r, reverse(split("/", var.private_dns_zone_id))[0]) == reverse(split("/", var.private_dns_zone_id))[0], false)])) - error_message = "Private DNS zone must be in one of the following format: `privatelink..azmk8s.io`, `.privatelink..azmk8s.io`, `private..azmk8s.io`, `.private..azmk8s.io`" + error_message = "According to the [document](https://learn.microsoft.com/en-us/azure/aks/private-clusters?tabs=azure-portal#configure-a-private-dns-zone), the private DNS zone must be in one of the following format: `privatelink..azmk8s.io`, `.privatelink..azmk8s.io`, `private..azmk8s.io`, `.private..azmk8s.io`" } } } From 6b6e5eaa417dd9f956255c0949fcdc435417552f Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 11:01:01 +0800 Subject: [PATCH 7/8] refactor check expression --- locals.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/locals.tf b/locals.tf index 08f55ed0..d1c0c4e5 100644 --- a/locals.tf +++ b/locals.tf @@ -54,6 +54,7 @@ locals { pool.pod_subnet_id ] ], [var.vnet_subnet_id])) + private_dns_zone_name = try(reverse(split("/", var.private_dns_zone_id))[0], null) query_datasource_for_log_analytics_workspace_location = var.log_analytics_workspace_enabled && (var.log_analytics_workspace != null ? var.log_analytics_workspace.location == null : false) subnet_ids = toset([for id in local.potential_subnet_ids : id if id != null]) use_brown_field_gw_for_ingress = var.brown_field_application_gateway_for_ingress != null From b1086b7ad254d033732160ee191dff2a53bb176e Mon Sep 17 00:00:00 2001 From: hezijie Date: Tue, 10 Sep 2024 11:07:00 +0800 Subject: [PATCH 8/8] refactor check expression --- main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.tf b/main.tf index 1570997c..6e7588ec 100644 --- a/main.tf +++ b/main.tf @@ -637,7 +637,7 @@ resource "azurerm_kubernetes_cluster" "main" { error_message = "A user assigned identity or a service principal must be used when using a custom private dns zone" } precondition { - condition = var.private_dns_zone_id == null ? true : (anytrue([for r in local.valid_private_dns_zone_regexs : try(regex(r, reverse(split("/", var.private_dns_zone_id))[0]) == reverse(split("/", var.private_dns_zone_id))[0], false)])) + condition = var.private_dns_zone_id == null ? true : (anytrue([for r in local.valid_private_dns_zone_regexs : try(regex(r, local.private_dns_zone_name) == local.private_dns_zone_name, false)])) error_message = "According to the [document](https://learn.microsoft.com/en-us/azure/aks/private-clusters?tabs=azure-portal#configure-a-private-dns-zone), the private DNS zone must be in one of the following format: `privatelink..azmk8s.io`, `.privatelink..azmk8s.io`, `private..azmk8s.io`, `.private..azmk8s.io`" } }