From b7b3833f11f3b47816e1bf161d8609e249626054 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 30 Jul 2024 14:00:00 +0800 Subject: [PATCH 1/8] `azurerm_kubernetes_cluster`, `azurerm_kubernetes_cluster_node_pool` - deprecate preview features --- .../kubernetes_cluster_data_source.go | 67 ++++--- .../kubernetes_cluster_data_source_test.go | 30 --- ...ubernetes_cluster_network_resource_test.go | 177 ------------------ .../kubernetes_cluster_node_pool_resource.go | 74 +++++--- ...ernetes_cluster_node_pool_resource_test.go | 65 ------- .../kubernetes_cluster_other_resource_test.go | 104 ---------- .../containers/kubernetes_cluster_resource.go | 141 ++++++++------ .../kubernetes_cluster_resource_test.go | 1 - .../containers/kubernetes_nodepool.go | 83 ++++---- .../docs/d/kubernetes_cluster.html.markdown | 4 - .../docs/r/kubernetes_cluster.html.markdown | 24 +-- ...kubernetes_cluster_node_pool.html.markdown | 10 +- 12 files changed, 217 insertions(+), 563 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 5dfb1036686f..13277a9e8f9b 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -324,14 +324,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { }, }, - "custom_ca_trust_certificates_base64": { - Type: pluginsdk.TypeList, - Computed: true, - Elem: &pluginsdk.Schema{ - Type: pluginsdk.TypeString, - }, - }, - "oms_agent": { Type: pluginsdk.TypeList, Computed: true, @@ -654,10 +646,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { Type: pluginsdk.TypeBool, Computed: true, }, - "disk_driver_version": { - Type: pluginsdk.TypeString, - Computed: true, - }, "file_driver_enabled": { Type: pluginsdk.TypeBool, Computed: true, @@ -745,6 +733,21 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { Computed: true, Deprecated: "This property is deprecated and will be removed in v4.0 of the AzureRM Provider in favour of the `node_public_ip_enabled` property.", } + resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeString, + Computed: true, + } + + resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeList, + Computed: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + }, + } + resource.Schema["azure_active_directory_role_based_access_control"] = &pluginsdk.Schema{ Type: pluginsdk.TypeList, Computed: true, @@ -865,9 +868,11 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} return fmt.Errorf("setting `key_management_service`: %+v", err) } - customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile) - if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil { - return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err) + if !features.FourPointOhBeta() { + customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile) + if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil { + return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err) + } } serviceMeshProfile := flattenKubernetesClusterAzureServiceMeshProfile(props.ServiceMeshProfile) @@ -1018,11 +1023,6 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man diskEnabled = *input.DiskCSIDriver.Enabled } - diskVersion := "" - if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil { - diskVersion = *input.DiskCSIDriver.Version - } - fileEnabled := true if input.FileCSIDriver != nil && input.FileCSIDriver.Enabled != nil { fileEnabled = *input.FileCSIDriver.Enabled @@ -1033,13 +1033,26 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man snapshotController = *input.SnapshotController.Enabled } - storageProfile = append(storageProfile, map[string]interface{}{ - "blob_driver_enabled": blobEnabled, - "disk_driver_enabled": diskEnabled, - "disk_driver_version": diskVersion, - "file_driver_enabled": fileEnabled, - "snapshot_controller_enabled": snapshotController, - }) + if !features.FourPointOhBeta() { + diskVersion := "" + if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil { + diskVersion = *input.DiskCSIDriver.Version + } + storageProfile = append(storageProfile, map[string]interface{}{ + "blob_driver_enabled": blobEnabled, + "disk_driver_enabled": diskEnabled, + "disk_driver_version": diskVersion, + "file_driver_enabled": fileEnabled, + "snapshot_controller_enabled": snapshotController, + }) + } else { + storageProfile = append(storageProfile, map[string]interface{}{ + "blob_driver_enabled": blobEnabled, + "disk_driver_enabled": diskEnabled, + "file_driver_enabled": fileEnabled, + "snapshot_controller_enabled": snapshotController, + }) + } } return storageProfile diff --git a/internal/services/containers/kubernetes_cluster_data_source_test.go b/internal/services/containers/kubernetes_cluster_data_source_test.go index 23b4d9b6935a..5aa33ee27b3d 100644 --- a/internal/services/containers/kubernetes_cluster_data_source_test.go +++ b/internal/services/containers/kubernetes_cluster_data_source_test.go @@ -554,26 +554,6 @@ func TestAccDataSourceKubernetesCluster_microsoftDefender(t *testing.T) { }) } -func TestAccDataSourceKubernetesCluster_customCaTrustCerts(t *testing.T) { - data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test") - r := KubernetesClusterDataSource{} - - fakeCertList := []string{ - "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzYKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", - "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmakNDQW1hZ0F3SUJBZ0lFZnlWdk56QU5CZ2txaGtpRzl3MEJBUXNGQURCWU1Rc3dDUVlEVlFRR0V3SlEKVERFVU1CSUdBMVVFQXd3TFJtRnJaU0JEWlhKMElESXhGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFYwpNQm9HQTFVRUNnd1RSR1ZtWVhWc2RDQkRiMjF3WVc1NUlFeDBaREFlRncweU16QTJNRFF3TnpJME1qZGFGdzB5Ck5UQTJNRE13TnpJME1qZGFNRmd4Q3pBSkJnTlZCQVlUQWxCTU1SUXdFZ1lEVlFRRERBdEdZV3RsSUVObGNuUWcKTWpFVk1CTUdBMVVFQnd3TVJHVm1ZWFZzZENCRGFYUjVNUnd3R2dZRFZRUUtEQk5FWldaaGRXeDBJRU52YlhCaApibmtnVEhSa01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMENTdVdUaGNjSG5MCkhFdjk4SUVNc2JLY3h4YVh4YTZiRXl1Yy9sUjRackpVN2p6eVlWNGVscTV5WTgwdDFCM0MyV3E2SXFoajErSGYKYW0xaStsU1FTejM1eWNnTWlwSWp2cUxKOVIzMVF0Wi9TRURkdGV2b2JqbytEa1dCOE55cG9Ia0pVbEIyQnR6ZgpOK09KeVFSdXU1b1cya2c5OE5Bd3JuTGpmQ0lremVWcFh5d0l4Tkx2ZmFrVGxpNWpYdG9WWG5pOTU5bmtINWVwClkrRnVoSEQwaU5CS25XYVkxR2QwVGhhSHNwTERmNFUycmo2WE5SZHd6QVZoVkdhUm02cndvSHRZeDVrYys1ZWMKQ0F4UEdRWFRzTzJUTHVrQzJ2YXI0M3RUM0ZjSC9taDRST2JaaThZS2xSQ3Fldm1QU1RmZ293RUFkTjlvSmxyRApXN2lzN2NnQjhRSURBUUFCbzFBd1RqQWRCZ05WSFE0RUZnUVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293Ckh3WURWUjBqQkJnd0ZvQVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293REFZRFZSMFRCQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBT0diT0Zyek4rN2YxbzhJSDNtMXZxT3IyTUtvNEZMWExGRjBVbEhkNApwZXRhL05aQjArUmQ3TnUrOCtnUnlUbEJWZU9EZjN5SXU0TlFCUU92MlNqdS9Jakd0MUtmaUF3WkUwT1RUQXc3CnhIWStsMVBJWEFFVWNqNk00cjFKQzc4ZVZrc2pycTZoV1RPZ0RrSVZuRjY3bXlReXduR25EY1k0d0Fqc2pUajgKKzR4NTIrRi9QaVNQVGtjUFNuN0s2UjQzaEt5QUs2Z0poOHE5cVNhME5RQ2U2czhwTGU2SVY5SElWVVFFVERVOQpsM1VWWHNBMGx4dlB0blU1TXo2QWQ5cDA5L2w4d3o0cUdBdGFCUEd3K0R2cTNlaHdTd2VZZ3VHSktDQjhjb01JCjJRVUo0Zi9mNkFNVWtMeWxYZ3RSUEt1QjA3d3YwTmk1eWI5MjlFY1FJQ0l2dFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t", - } - - data.DataSourceTest(t, []acceptance.TestStep{ - { - Config: r.customCaTrustCertificates(data, fakeCertList), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").Exists(), - ), - }, - }) -} - func TestAccDataSourceKubernetesCluster_serviceMesh(t *testing.T) { data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test") r := KubernetesClusterDataSource{} @@ -992,16 +972,6 @@ data "azurerm_kubernetes_cluster" "test" { `, KubernetesClusterResource{}.microsoftDefender(data)) } -func (KubernetesClusterDataSource) customCaTrustCertificates(data acceptance.TestData, fakeCertsList []string) string { - return fmt.Sprintf(` -%s -data "azurerm_kubernetes_cluster" "test" { - name = azurerm_kubernetes_cluster.test.name - resource_group_name = azurerm_kubernetes_cluster.test.resource_group_name -} -`, KubernetesClusterResource{}.customCATrustCertificates(data, fakeCertsList)) -} - func (KubernetesClusterDataSource) serviceMesh(data acceptance.TestData) string { return fmt.Sprintf(` %s diff --git a/internal/services/containers/kubernetes_cluster_network_resource_test.go b/internal/services/containers/kubernetes_cluster_network_resource_test.go index ec5d3a3c2085..2d738c9880c9 100644 --- a/internal/services/containers/kubernetes_cluster_network_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_network_resource_test.go @@ -978,36 +978,6 @@ func TestAccKubernetesCluster_networkDataPlane(t *testing.T) { }) } -func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") - r := KubernetesClusterResource{} - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.apiServerInManagedSubnet(data), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - }) -} - -func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") - r := KubernetesClusterResource{} - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.apiServerInBYOSubnet(data), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - }) -} - func TestAccKubernetesCluster_clusterPoolNodePublicIPTags(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -1067,153 +1037,6 @@ func TestAccKubernetesCluster_clusterPoolNetworkProfileUpdate(t *testing.T) { }) } -func (KubernetesClusterResource) apiServerInBYOSubnet(data acceptance.TestData) string { - return fmt.Sprintf(` -provider "azurerm" { - features {} -} - -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} - -resource "azurerm_virtual_network" "test" { - name = "acctestvirtnet%d" - address_space = ["10.0.0.0/8"] - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name -} - -resource "azurerm_subnet" "test" { - name = "acctestsubnet%d" - resource_group_name = azurerm_resource_group.test.name - virtual_network_name = azurerm_virtual_network.test.name - address_prefixes = ["10.1.0.0/16"] - - delegation { - name = "aks-delegation" - - service_delegation { - actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] - name = "Microsoft.ContainerService/managedClusters" - } - } -} - -resource "azurerm_subnet" "test1" { - name = "acctestsubnet1%d" - resource_group_name = azurerm_resource_group.test.name - virtual_network_name = azurerm_virtual_network.test.name - address_prefixes = ["10.2.0.0/16"] -} - -resource "azurerm_user_assigned_identity" "test" { - name = "acctestRG-aks-%d" - resource_group_name = azurerm_resource_group.test.name - location = azurerm_resource_group.test.location -} - -resource "azurerm_role_assignment" "test" { - scope = azurerm_subnet.test.id - role_definition_name = "Network Contributor" - principal_id = azurerm_user_assigned_identity.test.principal_id -} - -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - - api_server_access_profile { - vnet_integration_enabled = true - subnet_id = azurerm_subnet.test.id - } - - linux_profile { - admin_username = "acctestuser%d" - - ssh_key { - key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" - } - } - - default_node_pool { - name = "default" - node_count = 2 - vm_size = "Standard_DS2_v2" - vnet_subnet_id = azurerm_subnet.test1.id - upgrade_settings { - max_surge = "10%%" - } - } - - identity { - type = "UserAssigned" - identity_ids = [azurerm_user_assigned_identity.test.id] - } - - network_profile { - network_plugin = "azure" - } - - depends_on = [ - azurerm_role_assignment.test, - ] -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger) -} - -func (KubernetesClusterResource) apiServerInManagedSubnet(data acceptance.TestData) string { - return fmt.Sprintf(` -provider "azurerm" { - features {} -} - -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} - -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - - api_server_access_profile { - vnet_integration_enabled = true - } - - linux_profile { - admin_username = "acctestuser%d" - - ssh_key { - key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" - } - } - - default_node_pool { - name = "default" - node_count = 2 - vm_size = "Standard_DS2_v2" - upgrade_settings { - max_surge = "10%%" - } - } - - identity { - type = "SystemAssigned" - } - - network_profile { - network_plugin = "azure" - } -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger) -} - func (KubernetesClusterResource) advancedNetworkingConfig(data acceptance.TestData, networkPlugin string) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource.go b/internal/services/containers/kubernetes_cluster_node_pool_resource.go index 10427430ae5e..0fadcbb07415 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource.go @@ -134,11 +134,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { ValidateFunc: capacityreservationgroups.ValidateCapacityReservationGroupID, }, - "custom_ca_trust_enabled": { - Type: pluginsdk.TypeBool, - Optional: true, - }, - "eviction_policy": { Type: pluginsdk.TypeString, Optional: true, @@ -195,13 +190,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { ForceNew: true, }, - "message_of_the_day": { - Type: pluginsdk.TypeString, - Optional: true, - ForceNew: true, - ValidateFunc: validation.StringIsNotEmpty, - }, - "mode": { Type: pluginsdk.TypeString, Optional: true, @@ -390,7 +378,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { ValidateFunc: validation.StringInSlice([]string{ string(agentpools.WorkloadRuntimeOCIContainer), string(agentpools.WorkloadRuntimeWasmWasi), - string(agentpools.WorkloadRuntimeKataMshvVMIsolation), }, false), }, @@ -415,6 +402,20 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { } if !features.FourPointOhBeta() { + s["message_of_the_day"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringIsNotEmpty, + } + + s["custom_ca_trust_enabled"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeBool, + Optional: true, + } + s["os_sku"].ValidateFunc = validation.StringInSlice([]string{ string(agentpools.OSSKUAzureLinux), string(agentpools.OSSKUCBLMariner), @@ -423,6 +424,12 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { string(agentpools.OSSKUWindowsTwoZeroOneNine), string(agentpools.OSSKUWindowsTwoZeroTwoTwo), }, false) + + s["workload_runtime"].ValidateFunc = validation.StringInSlice([]string{ + string(agentpools.WorkloadRuntimeOCIContainer), + string(agentpools.WorkloadRuntimeWasmWasi), + string(agentpools.WorkloadRuntimeKataMshvVMIsolation), + }, false) } if !features.FourPointOh() { @@ -546,7 +553,6 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int profile := agentpools.ManagedClusterAgentPoolProfileProperties{ OsType: pointer.To(agentpools.OSType(osType)), EnableAutoScaling: pointer.To(enableAutoScaling), - EnableCustomCATrust: pointer.To(d.Get("custom_ca_trust_enabled").(bool)), EnableFIPS: pointer.To(d.Get("fips_enabled").(bool)), EnableEncryptionAtHost: pointer.To(hostEncryption), EnableUltraSSD: pointer.To(d.Get("ultra_ssd_enabled").(bool)), @@ -564,6 +570,10 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int Count: utils.Int64(int64(count)), } + if !features.FourPointOhBeta() { + profile.EnableCustomCATrust = pointer.To(d.Get("custom_ca_trust_enabled").(bool)) + } + if gpuInstanceProfile := d.Get("gpu_instance").(string); gpuInstanceProfile != "" { profile.GpuInstanceProfile = pointer.To(agentpools.GPUInstanceProfile(gpuInstanceProfile)) } @@ -625,12 +635,14 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int profile.NodeTaints = nodeTaints } - if v := d.Get("message_of_the_day").(string); v != "" { - if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows { - return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)") + if !features.FourPointOhBeta() { + if v := d.Get("message_of_the_day").(string); v != "" { + if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows { + return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)") + } + messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v)) + profile.MessageOfTheDay = &messageOfTheDayEncoded } - messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v)) - profile.MessageOfTheDay = &messageOfTheDayEncoded } if osDiskSizeGB := d.Get("os_disk_size_gb").(int); osDiskSizeGB > 0 { @@ -810,8 +822,10 @@ func resourceKubernetesClusterNodePoolUpdate(d *pluginsdk.ResourceData, meta int } } - if d.HasChange("custom_ca_trust_enabled") { - props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool)) + if !features.FourPointOhBeta() { + if d.HasChange("custom_ca_trust_enabled") { + props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool)) + } } if d.HasChange("max_count") || enableAutoScaling { @@ -973,12 +987,12 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter d.Set("auto_scaling_enabled", props.EnableAutoScaling) d.Set("node_public_ip_enabled", props.EnableNodePublicIP) d.Set("host_encryption_enabled", props.EnableEncryptionAtHost) + d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust) } else { d.Set("enable_auto_scaling", props.EnableAutoScaling) d.Set("enable_node_public_ip", props.EnableNodePublicIP) d.Set("enable_host_encryption", props.EnableEncryptionAtHost) } - d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust) d.Set("fips_enabled", props.EnableFIPS) d.Set("ultra_ssd_enabled", props.EnableUltraSSD) @@ -1028,15 +1042,17 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter } d.Set("max_count", maxCount) - messageOfTheDay := "" - if props.MessageOfTheDay != nil { - messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay) - if err != nil { - return fmt.Errorf("setting `message_of_the_day`: %+v", err) + if !features.FourPointOhBeta() { + messageOfTheDay := "" + if props.MessageOfTheDay != nil { + messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay) + if err != nil { + return fmt.Errorf("setting `message_of_the_day`: %+v", err) + } + messageOfTheDay = string(messageOfTheDayDecoded) } - messageOfTheDay = string(messageOfTheDayDecoded) + d.Set("message_of_the_day", messageOfTheDay) } - d.Set("message_of_the_day", messageOfTheDay) maxPods := 0 if props.MaxPods != nil { diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go b/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go index 7f39094f446b..cfcc79926375 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go @@ -949,35 +949,6 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) { ), }, data.ImportStep(), - { - Config: r.workloadRuntime(data, "KataMshvVmIsolation"), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - }) -} - -func TestAccKubernetesClusterNodePool_customCATrustEnabled(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test") - r := KubernetesClusterNodePoolResource{} - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.customCATrustEnabled(data, true), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - { - Config: r.customCATrustEnabled(data, false), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), }) } @@ -2436,7 +2407,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" { node_count = 3 fips_enabled = true kubelet_disk_type = "OS" - message_of_the_day = "daily message" } `, r.templateConfig(data)) } @@ -2676,41 +2646,6 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, workloadRuntime) } -func (KubernetesClusterNodePoolResource) customCATrustEnabled(data acceptance.TestData, enabled bool) string { - return fmt.Sprintf(` -provider "azurerm" { - features {} -} -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_D2s_v3" - upgrade_settings { - max_surge = "10%%" - } - } - identity { - type = "SystemAssigned" - } -} -resource "azurerm_kubernetes_cluster_node_pool" "test" { - name = "internal" - kubernetes_cluster_id = azurerm_kubernetes_cluster.test.id - vm_size = "Standard_D2s_v3" - custom_ca_trust_enabled = "%t" -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled) -} - func (KubernetesClusterNodePoolResource) windowsProfileOutboundNatEnabled(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_other_resource_test.go b/internal/services/containers/kubernetes_cluster_other_resource_test.go index db72950c0208..6afe92148939 100644 --- a/internal/services/containers/kubernetes_cluster_other_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_other_resource_test.go @@ -270,21 +270,6 @@ func TestAccKubernetesCluster_nodePoolOther(t *testing.T) { }) } -func TestAccKubernetesCluster_nodePoolKataMshvVmIsolation(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") - r := KubernetesClusterResource{} - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.nodePoolKataMshvVmIsolation(data), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - }) -} - func TestAccKubernetesCluster_upgradeSkuTier(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -913,28 +898,6 @@ func TestAccKubernetesCluster_workloadIdentity(t *testing.T) { }) } -func TestAccKubernetesCluster_customCATrustEnabled(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") - r := KubernetesClusterResource{} - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.customCATrustEnabled(data, true), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - { - Config: r.customCATrustEnabled(data, false), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - ), - }, - data.ImportStep(), - }) -} - func TestAccKubernetesCluster_webAppRoutingWithMultipleDnsZone(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -1879,7 +1842,6 @@ resource "azurerm_kubernetes_cluster" "test" { vm_size = "Standard_DS2_v2" fips_enabled = true kubelet_disk_type = "OS" - message_of_the_day = "daily message" workload_runtime = "OCIContainer" upgrade_settings { max_surge = "10%%" @@ -1893,42 +1855,6 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger) } -func (KubernetesClusterResource) nodePoolKataMshvVmIsolation(data acceptance.TestData) string { - return fmt.Sprintf(` -provider "azurerm" { - features {} -} - -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} - -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - - default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_D2s_v3" - message_of_the_day = "daily message" - os_sku = "AzureLinux" - workload_runtime = "KataMshvVmIsolation" - upgrade_settings { - max_surge = "10%%" - } - } - - identity { - type = "SystemAssigned" - } -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger) -} - func (KubernetesClusterResource) skuConfigStandard(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { @@ -3386,36 +3312,6 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomString, data.RandomInteger, data.RandomInteger) } -func (KubernetesClusterResource) customCATrustEnabled(data acceptance.TestData, enabled bool) string { - return fmt.Sprintf(` -provider "azurerm" { - features {} -} -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_D2s_v3" - custom_ca_trust_enabled = "%t" - upgrade_settings { - max_surge = "10%%" - } - } - identity { - type = "SystemAssigned" - } -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled) -} - func (KubernetesClusterResource) azureMonitorKubernetesMetricsEnabled(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 785c03beb0c1..015a4a023b51 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -122,7 +122,7 @@ func resourceKubernetesCluster() *pluginsdk.Resource { return old.(string) != "" }), pluginsdk.ForceNewIfChange("custom_ca_trust_certificates_base64", func(ctx context.Context, old, new, meta interface{}) bool { - return len(old.([]interface{})) > 0 && len(new.([]interface{})) == 0 + return !features.FourPointOhBeta() && len(old.([]interface{})) > 0 && len(new.([]interface{})) == 0 }), ), @@ -158,17 +158,6 @@ func resourceKubernetesCluster() *pluginsdk.Resource { MaxItems: 1, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ - "vnet_integration_enabled": { - Type: pluginsdk.TypeBool, - Optional: true, - }, - - "subnet_id": { - Type: pluginsdk.TypeString, - Optional: true, - ValidateFunc: commonids.ValidateSubnetID, - }, - "authorized_ip_ranges": { Type: pluginsdk.TypeSet, Optional: true, @@ -344,16 +333,6 @@ func resourceKubernetesCluster() *pluginsdk.Resource { Optional: true, }, - "custom_ca_trust_certificates_base64": { - Type: pluginsdk.TypeList, - Optional: true, - MaxItems: 10, - Elem: &pluginsdk.Schema{ - Type: pluginsdk.TypeString, - ValidateFunc: validation.StringIsBase64, - }, - }, - "default_node_pool": SchemaDefaultNodePool(), "disk_encryption_set_id": { @@ -1384,7 +1363,6 @@ func resourceKubernetesCluster() *pluginsdk.Resource { MaxItems: 1, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ - "blob_driver_enabled": { Type: pluginsdk.TypeBool, Optional: true, @@ -1395,15 +1373,6 @@ func resourceKubernetesCluster() *pluginsdk.Resource { Optional: true, Default: true, }, - "disk_driver_version": { - Type: pluginsdk.TypeString, - Optional: true, - Default: "v1", - ValidateFunc: validation.StringInSlice([]string{ - "v1", - "v2", - }, false), - }, "file_driver_enabled": { Type: pluginsdk.TypeBool, Optional: true, @@ -1536,6 +1505,37 @@ func resourceKubernetesCluster() *pluginsdk.Resource { Deprecated: "This property has been renamed to `authorized_ip_ranges` within the `api_server_access_profile` block and will be removed in v4.0 of the provider", ConflictsWith: []string{"api_server_access_profile.0.authorized_ip_ranges"}, } + resource.Schema["api_server_access_profile"].Elem.(*pluginsdk.Resource).Schema["vnet_integration_enabled"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeBool, + Optional: true, + } + resource.Schema["api_server_access_profile"].Elem.(*pluginsdk.Resource).Schema["subnet_id"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: commonids.ValidateSubnetID, + } + resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeList, + Optional: true, + MaxItems: 10, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + ValidateFunc: validation.StringIsBase64, + }, + } + resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeString, + Optional: true, + Default: "v1", + ValidateFunc: validation.StringInSlice([]string{ + "v1", + "v2", + }, false), + } resource.Schema["network_profile"].Elem.(*pluginsdk.Resource).Schema["docker_bridge_cidr"] = &pluginsdk.Schema{ Type: pluginsdk.TypeString, Optional: true, @@ -1980,8 +1980,10 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} autoUpgradeProfile.NodeOSUpgradeChannel = pointer.To(managedclusters.NodeOSUpgradeChannel(nodeOsChannelUpgrade)) } - if customCaTrustCertListRaw := d.Get("custom_ca_trust_certificates_base64").([]interface{}); len(customCaTrustCertListRaw) > 0 { - securityProfile.CustomCATrustCertificates = convertCustomCaTrustCertsInput(customCaTrustCertListRaw) + if !features.FourPointOhBeta() { + if customCaTrustCertListRaw := d.Get("custom_ca_trust_certificates_base64").([]interface{}); len(customCaTrustCertListRaw) > 0 { + securityProfile.CustomCATrustCertificates = convertCustomCaTrustCertsInput(customCaTrustCertListRaw) + } } parameters := managedclusters.ManagedCluster{ @@ -2535,7 +2537,7 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms } - if d.HasChanges("custom_ca_trust_certificates_base64") { + if !features.FourPointOhBeta() && d.HasChanges("custom_ca_trust_certificates_base64") { updateCluster = true customCaTrustCertListRaw := d.Get("custom_ca_trust_certificates_base64").([]interface{}) existing.Model.Properties.SecurityProfile.CustomCATrustCertificates = convertCustomCaTrustCertsInput(customCaTrustCertListRaw) @@ -2942,8 +2944,10 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("node_os_upgrade_channel", nodeOSUpgradeChannel) } - customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile) - d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList) + if !features.FourPointOhBeta() { + customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile) + d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList) + } enablePrivateCluster := false enablePrivateClusterPublicFQDN := false @@ -3381,17 +3385,19 @@ func expandKubernetesClusterAPIAccessProfile(d *pluginsdk.ResourceData) *managed } } - enableVnetIntegration := false - if v := config["vnet_integration_enabled"]; v != nil { - enableVnetIntegration = v.(bool) - } - apiAccessProfile.EnableVnetIntegration = utils.Bool(enableVnetIntegration) + if !features.FourPointOhBeta() { + enableVnetIntegration := false + if v := config["vnet_integration_enabled"]; v != nil { + enableVnetIntegration = v.(bool) + } + apiAccessProfile.EnableVnetIntegration = utils.Bool(enableVnetIntegration) - subnetId := "" - if v := config["subnet_id"]; v != nil { - subnetId = v.(string) + subnetId := "" + if v := config["subnet_id"]; v != nil { + subnetId = v.(string) + } + apiAccessProfile.SubnetId = utils.String(subnetId) } - apiAccessProfile.SubnetId = utils.String(subnetId) return apiAccessProfile } @@ -3400,26 +3406,40 @@ func flattenKubernetesClusterAPIAccessProfile(profile *managedclusters.ManagedCl // some properties in this block are exposed within the `api_server_access_profile` block and others are exposed as // top level properties which causes strange diffs depending on what is being set, so this also needs to check // whether the properties in the block are returned or nil - if profile == nil || (profile.AuthorizedIPRanges == nil && profile.SubnetId == nil && profile.EnableVnetIntegration == nil) { - return []interface{}{} + if !features.FourPointOhBeta() { + if profile == nil || (profile.AuthorizedIPRanges == nil && profile.SubnetId == nil && profile.EnableVnetIntegration == nil) { + return []interface{}{} + } + } else { + if profile == nil || profile.AuthorizedIPRanges == nil { + return []interface{}{} + } } apiServerAuthorizedIPRanges := utils.FlattenStringSlice(profile.AuthorizedIPRanges) - enableVnetIntegration := false - if profile.EnableVnetIntegration != nil { - enableVnetIntegration = *profile.EnableVnetIntegration - } - subnetId := "" - if profile.SubnetId != nil && *profile.SubnetId != "" { - subnetId = *profile.SubnetId + if !features.FourPointOhBeta() { + enableVnetIntegration := false + if profile.EnableVnetIntegration != nil { + enableVnetIntegration = *profile.EnableVnetIntegration + } + subnetId := "" + if profile.SubnetId != nil && *profile.SubnetId != "" { + subnetId = *profile.SubnetId + } + + return []interface{}{ + map[string]interface{}{ + "authorized_ip_ranges": apiServerAuthorizedIPRanges, + "subnet_id": subnetId, + "vnet_integration_enabled": enableVnetIntegration, + }, + } } return []interface{}{ map[string]interface{}{ - "authorized_ip_ranges": apiServerAuthorizedIPRanges, - "subnet_id": subnetId, - "vnet_integration_enabled": enableVnetIntegration, + "authorized_ip_ranges": apiServerAuthorizedIPRanges, }, } } @@ -4768,7 +4788,6 @@ func expandStorageProfile(input []interface{}) *managedclusters.ManagedClusterSt }, DiskCSIDriver: &managedclusters.ManagedClusterStorageProfileDiskCSIDriver{ Enabled: utils.Bool(raw["disk_driver_enabled"].(bool)), - Version: utils.String(raw["disk_driver_version"].(string)), }, FileCSIDriver: &managedclusters.ManagedClusterStorageProfileFileCSIDriver{ Enabled: utils.Bool(raw["file_driver_enabled"].(bool)), @@ -4778,6 +4797,10 @@ func expandStorageProfile(input []interface{}) *managedclusters.ManagedClusterSt }, } + if !features.FourPointOhBeta() { + profile.DiskCSIDriver.Version = utils.String(raw["disk_driver_version"].(string)) + } + return &profile } diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 40085cd72be0..6c24fb653a80 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -884,7 +884,6 @@ resource "azurerm_kubernetes_cluster" "test" { storage_profile { blob_driver_enabled = true disk_driver_enabled = true - disk_driver_version = "v1" file_driver_enabled = false snapshot_controller_enabled = false } diff --git a/internal/services/containers/kubernetes_nodepool.go b/internal/services/containers/kubernetes_nodepool.go index dfccb20ba8c5..97cf2acaaaa4 100644 --- a/internal/services/containers/kubernetes_nodepool.go +++ b/internal/services/containers/kubernetes_nodepool.go @@ -78,11 +78,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { ValidateFunc: capacityreservationgroups.ValidateCapacityReservationGroupID, }, - "custom_ca_trust_enabled": { - Type: pluginsdk.TypeBool, - Optional: true, - }, - "kubelet_config": schemaNodePoolKubeletConfig(), "linux_os_config": schemaNodePoolLinuxOSConfig(), @@ -129,13 +124,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { Computed: true, }, - "message_of_the_day": { - Type: pluginsdk.TypeString, - Optional: true, - ForceNew: true, - ValidateFunc: validation.StringIsNotEmpty, - }, - "min_count": { Type: pluginsdk.TypeInt, Optional: true, @@ -269,7 +257,6 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { Computed: true, ValidateFunc: validation.StringInSlice([]string{ string(managedclusters.WorkloadRuntimeOCIContainer), - string(managedclusters.WorkloadRuntimeKataMshvVMIsolation), }, false), }, @@ -292,6 +279,20 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { } if !features.FourPointOhBeta() { + s["custom_ca_trust_enabled"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeBool, + Optional: true, + } + + s["message_of_the_day"] = &pluginsdk.Schema{ + Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Type: pluginsdk.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validation.StringIsNotEmpty, + } + s["os_sku"].ValidateFunc = validation.StringInSlice([]string{ string(agentpools.OSSKUAzureLinux), string(agentpools.OSSKUCBLMariner), @@ -301,6 +302,11 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { string(agentpools.OSSKUWindowsTwoZeroTwoTwo), }, false) + s["workload_runtime"].ValidateFunc = validation.StringInSlice([]string{ + string(managedclusters.WorkloadRuntimeOCIContainer), + string(managedclusters.WorkloadRuntimeKataMshvVMIsolation), + }, false) + s["node_taints"] = &pluginsdk.Schema{ Type: pluginsdk.TypeList, Optional: true, @@ -1076,10 +1082,8 @@ func ConvertDefaultNodePoolToAgentPool(input *[]managedclusters.ManagedClusterAg VnetSubnetID: defaultCluster.VnetSubnetID, MaxPods: defaultCluster.MaxPods, MaxCount: defaultCluster.MaxCount, - MessageOfTheDay: defaultCluster.MessageOfTheDay, MinCount: defaultCluster.MinCount, EnableAutoScaling: defaultCluster.EnableAutoScaling, - EnableCustomCATrust: defaultCluster.EnableCustomCATrust, EnableEncryptionAtHost: defaultCluster.EnableEncryptionAtHost, EnableFIPS: defaultCluster.EnableFIPS, EnableUltraSSD: defaultCluster.EnableUltraSSD, @@ -1095,6 +1099,12 @@ func ConvertDefaultNodePoolToAgentPool(input *[]managedclusters.ManagedClusterAg Tags: defaultCluster.Tags, }, } + + if !features.FourPointOhBeta() { + agentpool.Properties.MessageOfTheDay = defaultCluster.MessageOfTheDay + agentpool.Properties.EnableCustomCATrust = defaultCluster.EnableCustomCATrust + } + if osDisktypeNodePool := defaultCluster.OsDiskType; osDisktypeNodePool != nil { osDisktype := agentpools.OSDiskType(string(*osDisktypeNodePool)) agentpool.Properties.OsDiskType = &osDisktype @@ -1253,7 +1263,6 @@ func ExpandDefaultNodePool(d *pluginsdk.ResourceData) (*[]managedclusters.Manage profile := managedclusters.ManagedClusterAgentPoolProfile{ EnableAutoScaling: utils.Bool(enableAutoScaling), - EnableCustomCATrust: utils.Bool(raw["custom_ca_trust_enabled"].(bool)), EnableFIPS: utils.Bool(raw["fips_enabled"].(bool)), EnableNodePublicIP: utils.Bool(nodePublicIp), EnableEncryptionAtHost: utils.Bool(hostEncryption), @@ -1291,9 +1300,13 @@ func ExpandDefaultNodePool(d *pluginsdk.ResourceData) (*[]managedclusters.Manage profile.MaxPods = utils.Int64(maxPods) } - if v := raw["message_of_the_day"].(string); v != "" { - messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v)) - profile.MessageOfTheDay = &messageOfTheDayEncoded + if !features.FourPointOhBeta() { + if v := raw["message_of_the_day"].(string); v != "" { + messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v)) + profile.MessageOfTheDay = &messageOfTheDayEncoded + } + + profile.EnableCustomCATrust = utils.Bool(raw["custom_ca_trust_enabled"].(bool)) } if prefixID := raw["node_public_ip_prefix_id"].(string); prefixID != "" { @@ -1620,11 +1633,6 @@ func FlattenDefaultNodePool(input *[]managedclusters.ManagedClusterAgentPoolProf enableAutoScaling = *agentPool.EnableAutoScaling } - customCaTrustEnabled := false - if agentPool.EnableCustomCATrust != nil { - customCaTrustEnabled = *agentPool.EnableCustomCATrust - } - enableFIPS := false if agentPool.EnableFIPS != nil { enableFIPS = *agentPool.EnableFIPS @@ -1655,15 +1663,6 @@ func FlattenDefaultNodePool(input *[]managedclusters.ManagedClusterAgentPoolProf maxPods = int(*agentPool.MaxPods) } - messageOfTheDay := "" - if agentPool.MessageOfTheDay != nil { - messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*agentPool.MessageOfTheDay) - if err != nil { - return nil, err - } - messageOfTheDay = string(messageOfTheDayDecoded) - } - minCount := 0 if agentPool.MinCount != nil { minCount = int(*agentPool.MinCount) @@ -1786,14 +1785,12 @@ func FlattenDefaultNodePool(input *[]managedclusters.ManagedClusterAgentPoolProf networkProfile := flattenClusterPoolNetworkProfile(agentPool.NetworkProfile) out := map[string]interface{}{ - "custom_ca_trust_enabled": customCaTrustEnabled, "fips_enabled": enableFIPS, "gpu_instance": gpuInstanceProfile, "host_group_id": hostGroupID, "kubelet_disk_type": kubeletDiskType, "max_count": maxCount, "max_pods": maxPods, - "message_of_the_day": messageOfTheDay, "min_count": minCount, "name": name, "node_count": count, @@ -1824,7 +1821,23 @@ func FlattenDefaultNodePool(input *[]managedclusters.ManagedClusterAgentPoolProf } if !features.FourPointOhBeta() { + customCaTrustEnabled := false + if agentPool.EnableCustomCATrust != nil { + customCaTrustEnabled = *agentPool.EnableCustomCATrust + } + + messageOfTheDay := "" + if agentPool.MessageOfTheDay != nil { + messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*agentPool.MessageOfTheDay) + if err != nil { + return nil, err + } + messageOfTheDay = string(messageOfTheDayDecoded) + } + out["node_taints"] = []string{} + out["custom_ca_trust_enabled"] = customCaTrustEnabled + out["message_of_the_day"] = messageOfTheDay } if features.FourPointOh() { diff --git a/website/docs/d/kubernetes_cluster.html.markdown b/website/docs/d/kubernetes_cluster.html.markdown index 156579b446a1..1985203f440d 100644 --- a/website/docs/d/kubernetes_cluster.html.markdown +++ b/website/docs/d/kubernetes_cluster.html.markdown @@ -114,8 +114,6 @@ The following attributes are exported: * `tags` - A mapping of tags assigned to this resource. -* `custom_ca_trust_certificates_base64` - A list of custom base64 encoded CAs used by this Managed Kubernetes Cluster. - --- An `aci_connector_linux` block exports the following: @@ -341,8 +339,6 @@ A `storage_profile` block exports the following: * `disk_driver_enabled` Is the Disk CSI driver enabled? -* `disk_driver_version` The configured Disk CSI Driver version. - * `file_driver_enabled` Is the File CSI driver enabled? * `snapshot_controller_enabled` Is the Snapshot Controller enabled? diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 891137999d06..bc488d45dd9b 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -101,10 +101,6 @@ In addition, one of either `identity` or `service_principal` blocks must be spec * `cost_analysis_enabled` - (Optional) Should cost analysis be enabled for this Kubernetes Cluster? Defaults to `false`. The `sku_tier` must be set to `Standard` or `Premium` to enable this feature. Enabling this will add Kubernetes Namespace and Deployment details to the Cost Analysis views in the Azure portal. -* `custom_ca_trust_certificates_base64` - (Optional) A list of up to 10 base64 encoded CAs that will be added to the trust store on nodes with the `custom_ca_trust_enabled` feature enabled. - --> **Note:** Removing `custom_ca_trust_certificates_base64` after it has been set forces a new resource to be created. - * `disk_encryption_set_id` - (Optional) The ID of the Disk Encryption Set which should be used for the Nodes and Volumes. More information [can be found in the documentation](https://docs.microsoft.com/azure/aks/azure-disk-customer-managed-keys). Changing this forces a new resource to be created. * `edge_zone` - (Optional) Specifies the Edge Zone within the Azure Region where this Managed Kubernetes Cluster should exist. Changing this forces a new resource to be created. @@ -284,12 +280,6 @@ An `api_server_access_profile` block supports the following: * `authorized_ip_ranges` - (Optional) Set of authorized IP ranges to allow access to API server, e.g. ["198.51.100.0/24"]. -* `subnet_id` - (Optional) The ID of the Subnet where the API server endpoint is delegated to. - -* `vnet_integration_enabled` - (Optional) Should API Server VNet Integration be enabled? For more details please visit [Use API Server VNet Integration](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration). - --> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/EnableAPIServerVnetIntegrationPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/api-server-vnet-integration#register-the-enableapiservervnetintegrationpreview-preview-feature) for more information. - --- An `auto_scaler_profile` block supports the following: @@ -372,10 +362,6 @@ A `default_node_pool` block supports the following: * `capacity_reservation_group_id` - (Optional) Specifies the ID of the Capacity Reservation Group within which this AKS Cluster should be created. Changing this forces a new resource to be created. -* `custom_ca_trust_enabled` - (Optional) Specifies whether to trust a Custom CA. - --> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/CustomCATrustPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/custom-certificate-authority) for more information. - * `enable_auto_scaling` - (Optional) Should [the Kubernetes Auto Scaler](https://docs.microsoft.com/azure/aks/cluster-autoscaler) be enabled for this Node Pool? -> **Note:** This requires that the `type` is set to `VirtualMachineScaleSets`. @@ -402,8 +388,6 @@ A `default_node_pool` block supports the following: * `max_pods` - (Optional) The maximum number of pods that can run on each agent. `temporary_name_for_rotation` must be specified when changing this property. -* `message_of_the_day` - (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created. - * `node_network_profile` - (Optional) A `node_network_profile` block as documented below. * `node_public_ip_prefix_id` - (Optional) Resource ID for the Public IP Addresses Prefix for the nodes in this Node Pool. `enable_node_public_ip` should be `true`. Changing this forces a new resource to be created. @@ -448,9 +432,7 @@ A `default_node_pool` block supports the following: ~> **Note:** A Route Table must be configured on this Subnet. -* `workload_runtime` - (Optional) Specifies the workload runtime used by the node pool. Possible values are `OCIContainer` and `KataMshvVmIsolation`. - -~> **Note:** Pod Sandboxing / KataVM Isolation node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://learn.microsoft.com/azure/aks/use-pod-sandboxing) +* `workload_runtime` - (Optional) Specifies the workload runtime used by the node pool. Possible value is `OCIContainer`. * `zones` - (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster should be located. `temporary_name_for_rotation` must be specified when changing this property. @@ -825,10 +807,6 @@ A `storage_profile` block supports the following: * `disk_driver_enabled` - (Optional) Is the Disk CSI driver enabled? Defaults to `true`. -* `disk_driver_version` - (Optional) Disk CSI Driver version to be used. Possible values are `v1` and `v2`. Defaults to `v1`. - --> **Note:** `Azure Disk CSI driver v2` is currently in [Public Preview](https://azure.microsoft.com/en-us/updates/public-preview-azure-disk-csi-driver-v2-in-aks/) on an opt-in basis. To use it, the feature `EnableAzureDiskCSIDriverV2` for namespace `Microsoft.ContainerService` must be requested. - * `file_driver_enabled` - (Optional) Is the File CSI driver enabled? Defaults to `true`. * `snapshot_controller_enabled` - (Optional) Is the Snapshot Controller enabled? Defaults to `true`. diff --git a/website/docs/r/kubernetes_cluster_node_pool.html.markdown b/website/docs/r/kubernetes_cluster_node_pool.html.markdown index 333a64b4c3f5..ffd20bd6b855 100644 --- a/website/docs/r/kubernetes_cluster_node_pool.html.markdown +++ b/website/docs/r/kubernetes_cluster_node_pool.html.markdown @@ -72,10 +72,6 @@ The following arguments are supported: * `capacity_reservation_group_id` - (Optional) Specifies the ID of the Capacity Reservation Group where this Node Pool should exist. Changing this forces a new resource to be created. -* `custom_ca_trust_enabled` - (Optional) Specifies whether to trust a Custom CA. - --> **Note:** This requires that the Preview Feature `Microsoft.ContainerService/CustomCATrustPreview` is enabled and the Resource Provider is re-registered, see [the documentation](https://learn.microsoft.com/en-us/azure/aks/custom-certificate-authority) for more information. - * `enable_auto_scaling` - (Optional) Whether to enable [auto-scaler](https://docs.microsoft.com/azure/aks/cluster-autoscaler). * `enable_host_encryption` - (Optional) Should the nodes in this Node Pool have host encryption enabled? Changing this forces a new resource to be created. @@ -104,8 +100,6 @@ The following arguments are supported: * `max_pods` - (Optional) The maximum number of pods that can run on each agent. Changing this forces a new resource to be created. -* `message_of_the_day` - (Optional) A base64-encoded string which will be written to /etc/motd after decoding. This allows customization of the message of the day for Linux nodes. It cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script). Changing this forces a new resource to be created. - * `mode` - (Optional) Should this Node Pool be used for System or User resources? Possible values are `System` and `User`. Defaults to `User`. * `node_network_profile` - (Optional) A `node_network_profile` block as documented below. @@ -158,12 +152,10 @@ The following arguments are supported: * `windows_profile` - (Optional) A `windows_profile` block as documented below. Changing this forces a new resource to be created. -* `workload_runtime` - (Optional) Used to specify the workload runtime. Allowed values are `OCIContainer`, `WasmWasi` and `KataMshvVmIsolation`. +* `workload_runtime` - (Optional) Used to specify the workload runtime. Allowed values are `OCIContainer` and `WasmWasi`. ~> **Note:** WebAssembly System Interface node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://docs.microsoft.com/azure/aks/use-wasi-node-pools) -~> **Note:** Pod Sandboxing / KataVM Isolation node pools are in Public Preview - more information and details on how to opt into the preview can be found in [this article](https://learn.microsoft.com/azure/aks/use-pod-sandboxing) - * `zones` - (Optional) Specifies a list of Availability Zones in which this Kubernetes Cluster Node Pool should be located. Changing this forces a new Kubernetes Cluster Node Pool to be created. --- From 30998ffcad10e3c12f5ef8cc3dbd4ee3c579b35a Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 30 Jul 2024 15:08:27 +0800 Subject: [PATCH 2/8] make terrafmt --- .../kubernetes_cluster_other_resource_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_other_resource_test.go b/internal/services/containers/kubernetes_cluster_other_resource_test.go index 6afe92148939..489aaccc1a12 100644 --- a/internal/services/containers/kubernetes_cluster_other_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_other_resource_test.go @@ -1837,12 +1837,12 @@ resource "azurerm_kubernetes_cluster" "test" { dns_prefix = "acctestaks%d" default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_DS2_v2" - fips_enabled = true - kubelet_disk_type = "OS" - workload_runtime = "OCIContainer" + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + fips_enabled = true + kubelet_disk_type = "OS" + workload_runtime = "OCIContainer" upgrade_settings { max_surge = "10%%" } From 5a2fa727cdb1cbaab56010ba7760a601e303a124 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 30 Jul 2024 16:50:40 +0800 Subject: [PATCH 3/8] only set custom_ca_trust_enabled if it's not 4.0 for node pool --- .../containers/kubernetes_cluster_node_pool_resource.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource.go b/internal/services/containers/kubernetes_cluster_node_pool_resource.go index 0fadcbb07415..4cf714ad072e 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource.go @@ -987,11 +987,11 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter d.Set("auto_scaling_enabled", props.EnableAutoScaling) d.Set("node_public_ip_enabled", props.EnableNodePublicIP) d.Set("host_encryption_enabled", props.EnableEncryptionAtHost) - d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust) } else { d.Set("enable_auto_scaling", props.EnableAutoScaling) d.Set("enable_node_public_ip", props.EnableNodePublicIP) d.Set("enable_host_encryption", props.EnableEncryptionAtHost) + d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust) } d.Set("fips_enabled", props.EnableFIPS) d.Set("ultra_ssd_enabled", props.EnableUltraSSD) From a08406c1be83dc0d7438e40327d38c87323de33f Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Wed, 31 Jul 2024 09:35:09 +0800 Subject: [PATCH 4/8] remove tests --- .../kubernetes_cluster_other_resource_test.go | 73 ------------------- 1 file changed, 73 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_other_resource_test.go b/internal/services/containers/kubernetes_cluster_other_resource_test.go index 489aaccc1a12..a98144d837ce 100644 --- a/internal/services/containers/kubernetes_cluster_other_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_other_resource_test.go @@ -1049,41 +1049,6 @@ func TestAccKubernetesCluster_nodeOsUpgradeChannel(t *testing.T) { }) } -func TestAccKubernetesCluster_customCaTrustCerts(t *testing.T) { - data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") - r := KubernetesClusterResource{} - - fakeCertList := []string{ - "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzYKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", - "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmakNDQW1hZ0F3SUJBZ0lFZnlWdk56QU5CZ2txaGtpRzl3MEJBUXNGQURCWU1Rc3dDUVlEVlFRR0V3SlEKVERFVU1CSUdBMVVFQXd3TFJtRnJaU0JEWlhKMElESXhGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFYwpNQm9HQTFVRUNnd1RSR1ZtWVhWc2RDQkRiMjF3WVc1NUlFeDBaREFlRncweU16QTJNRFF3TnpJME1qZGFGdzB5Ck5UQTJNRE13TnpJME1qZGFNRmd4Q3pBSkJnTlZCQVlUQWxCTU1SUXdFZ1lEVlFRRERBdEdZV3RsSUVObGNuUWcKTWpFVk1CTUdBMVVFQnd3TVJHVm1ZWFZzZENCRGFYUjVNUnd3R2dZRFZRUUtEQk5FWldaaGRXeDBJRU52YlhCaApibmtnVEhSa01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMENTdVdUaGNjSG5MCkhFdjk4SUVNc2JLY3h4YVh4YTZiRXl1Yy9sUjRackpVN2p6eVlWNGVscTV5WTgwdDFCM0MyV3E2SXFoajErSGYKYW0xaStsU1FTejM1eWNnTWlwSWp2cUxKOVIzMVF0Wi9TRURkdGV2b2JqbytEa1dCOE55cG9Ia0pVbEIyQnR6ZgpOK09KeVFSdXU1b1cya2c5OE5Bd3JuTGpmQ0lremVWcFh5d0l4Tkx2ZmFrVGxpNWpYdG9WWG5pOTU5bmtINWVwClkrRnVoSEQwaU5CS25XYVkxR2QwVGhhSHNwTERmNFUycmo2WE5SZHd6QVZoVkdhUm02cndvSHRZeDVrYys1ZWMKQ0F4UEdRWFRzTzJUTHVrQzJ2YXI0M3RUM0ZjSC9taDRST2JaaThZS2xSQ3Fldm1QU1RmZ293RUFkTjlvSmxyRApXN2lzN2NnQjhRSURBUUFCbzFBd1RqQWRCZ05WSFE0RUZnUVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293Ckh3WURWUjBqQkJnd0ZvQVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293REFZRFZSMFRCQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBT0diT0Zyek4rN2YxbzhJSDNtMXZxT3IyTUtvNEZMWExGRjBVbEhkNApwZXRhL05aQjArUmQ3TnUrOCtnUnlUbEJWZU9EZjN5SXU0TlFCUU92MlNqdS9Jakd0MUtmaUF3WkUwT1RUQXc3CnhIWStsMVBJWEFFVWNqNk00cjFKQzc4ZVZrc2pycTZoV1RPZ0RrSVZuRjY3bXlReXduR25EY1k0d0Fqc2pUajgKKzR4NTIrRi9QaVNQVGtjUFNuN0s2UjQzaEt5QUs2Z0poOHE5cVNhME5RQ2U2czhwTGU2SVY5SElWVVFFVERVOQpsM1VWWHNBMGx4dlB0blU1TXo2QWQ5cDA5L2w4d3o0cUdBdGFCUEd3K0R2cTNlaHdTd2VZZ3VHSktDQjhjb01JCjJRVUo0Zi9mNkFNVWtMeWxYZ3RSUEt1QjA3d3YwTmk1eWI5MjlFY1FJQ0l2dFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t", - } - - fakeCertList2 := []string{ - "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzcKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", - } - - data.ResourceTest(t, r, []acceptance.TestStep{ - { - Config: r.customCATrustCertificates(data, fakeCertList), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").Exists(), - ), - }, - data.ImportStep(), - { - Config: r.customCATrustCertificates(data, fakeCertList2), - Check: acceptance.ComposeTestCheckFunc( - check.That(data.ResourceName).ExistsInAzure(r), - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), - check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").DoesNotExist(), - ), - }, - data.ImportStep(), - }) -} - func TestAccKubernetesCluster_snapshotId(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -3478,44 +3443,6 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, nodeOsUpgradeChannel) } -func (KubernetesClusterResource) customCATrustCertificates(data acceptance.TestData, certsList []string) string { - - certsString := "" - - if certsList != nil { - certsString = "\"" + strings.Join(certsList, "\" ,\"") + "\"" - } - - return fmt.Sprintf(` -provider "azurerm" { - features {} -} -resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" -} -resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - role_based_access_control_enabled = true - default_node_pool { - name = "default" - node_count = 1 - vm_size = "Standard_DS2_v2" - upgrade_settings { - max_surge = "10%%" - } - } - identity { - type = "SystemAssigned" - } - custom_ca_trust_certificates_base64 = [%s] -} -`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, certsString) -} - func (KubernetesClusterResource) snapshotSource(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { From 4fd4f6b4881ca75675d0accd571c09bc407c18e7 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Thu, 1 Aug 2024 16:43:27 +0800 Subject: [PATCH 5/8] add skip on the tests --- .../kubernetes_cluster_data_source_test.go | 33 +++ ...ubernetes_cluster_network_resource_test.go | 159 +++++++++++++ ...ernetes_cluster_node_pool_resource_test.go | 99 ++++++++ .../kubernetes_cluster_other_resource_test.go | 221 ++++++++++++++++++ .../kubernetes_cluster_resource_test.go | 42 ++++ 5 files changed, 554 insertions(+) diff --git a/internal/services/containers/kubernetes_cluster_data_source_test.go b/internal/services/containers/kubernetes_cluster_data_source_test.go index 5aa33ee27b3d..0f87c4f37a17 100644 --- a/internal/services/containers/kubernetes_cluster_data_source_test.go +++ b/internal/services/containers/kubernetes_cluster_data_source_test.go @@ -554,6 +554,29 @@ func TestAccDataSourceKubernetesCluster_microsoftDefender(t *testing.T) { }) } +func TestAccDataSourceKubernetesCluster_customCaTrustCerts(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test") + r := KubernetesClusterDataSource{} + + fakeCertList := []string{ + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzYKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmakNDQW1hZ0F3SUJBZ0lFZnlWdk56QU5CZ2txaGtpRzl3MEJBUXNGQURCWU1Rc3dDUVlEVlFRR0V3SlEKVERFVU1CSUdBMVVFQXd3TFJtRnJaU0JEWlhKMElESXhGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFYwpNQm9HQTFVRUNnd1RSR1ZtWVhWc2RDQkRiMjF3WVc1NUlFeDBaREFlRncweU16QTJNRFF3TnpJME1qZGFGdzB5Ck5UQTJNRE13TnpJME1qZGFNRmd4Q3pBSkJnTlZCQVlUQWxCTU1SUXdFZ1lEVlFRRERBdEdZV3RsSUVObGNuUWcKTWpFVk1CTUdBMVVFQnd3TVJHVm1ZWFZzZENCRGFYUjVNUnd3R2dZRFZRUUtEQk5FWldaaGRXeDBJRU52YlhCaApibmtnVEhSa01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMENTdVdUaGNjSG5MCkhFdjk4SUVNc2JLY3h4YVh4YTZiRXl1Yy9sUjRackpVN2p6eVlWNGVscTV5WTgwdDFCM0MyV3E2SXFoajErSGYKYW0xaStsU1FTejM1eWNnTWlwSWp2cUxKOVIzMVF0Wi9TRURkdGV2b2JqbytEa1dCOE55cG9Ia0pVbEIyQnR6ZgpOK09KeVFSdXU1b1cya2c5OE5Bd3JuTGpmQ0lremVWcFh5d0l4Tkx2ZmFrVGxpNWpYdG9WWG5pOTU5bmtINWVwClkrRnVoSEQwaU5CS25XYVkxR2QwVGhhSHNwTERmNFUycmo2WE5SZHd6QVZoVkdhUm02cndvSHRZeDVrYys1ZWMKQ0F4UEdRWFRzTzJUTHVrQzJ2YXI0M3RUM0ZjSC9taDRST2JaaThZS2xSQ3Fldm1QU1RmZ293RUFkTjlvSmxyRApXN2lzN2NnQjhRSURBUUFCbzFBd1RqQWRCZ05WSFE0RUZnUVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293Ckh3WURWUjBqQkJnd0ZvQVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293REFZRFZSMFRCQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBT0diT0Zyek4rN2YxbzhJSDNtMXZxT3IyTUtvNEZMWExGRjBVbEhkNApwZXRhL05aQjArUmQ3TnUrOCtnUnlUbEJWZU9EZjN5SXU0TlFCUU92MlNqdS9Jakd0MUtmaUF3WkUwT1RUQXc3CnhIWStsMVBJWEFFVWNqNk00cjFKQzc4ZVZrc2pycTZoV1RPZ0RrSVZuRjY3bXlReXduR25EY1k0d0Fqc2pUajgKKzR4NTIrRi9QaVNQVGtjUFNuN0s2UjQzaEt5QUs2Z0poOHE5cVNhME5RQ2U2czhwTGU2SVY5SElWVVFFVERVOQpsM1VWWHNBMGx4dlB0blU1TXo2QWQ5cDA5L2w4d3o0cUdBdGFCUEd3K0R2cTNlaHdTd2VZZ3VHSktDQjhjb01JCjJRVUo0Zi9mNkFNVWtMeWxYZ3RSUEt1QjA3d3YwTmk1eWI5MjlFY1FJQ0l2dFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t", + } + + data.DataSourceTest(t, []acceptance.TestStep{ + { + Config: r.customCaTrustCertificates(data, fakeCertList), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").Exists(), + ), + }, + }) +} + func TestAccDataSourceKubernetesCluster_serviceMesh(t *testing.T) { data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test") r := KubernetesClusterDataSource{} @@ -972,6 +995,16 @@ data "azurerm_kubernetes_cluster" "test" { `, KubernetesClusterResource{}.microsoftDefender(data)) } +func (KubernetesClusterDataSource) customCaTrustCertificates(data acceptance.TestData, fakeCertsList []string) string { + return fmt.Sprintf(` +%s +data "azurerm_kubernetes_cluster" "test" { + name = azurerm_kubernetes_cluster.test.name + resource_group_name = azurerm_kubernetes_cluster.test.resource_group_name +} +`, KubernetesClusterResource{}.customCATrustCertificates(data, fakeCertsList)) +} + func (KubernetesClusterDataSource) serviceMesh(data acceptance.TestData) string { return fmt.Sprintf(` %s diff --git a/internal/services/containers/kubernetes_cluster_network_resource_test.go b/internal/services/containers/kubernetes_cluster_network_resource_test.go index 2d738c9880c9..0a39d70a7103 100644 --- a/internal/services/containers/kubernetes_cluster_network_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_network_resource_test.go @@ -978,6 +978,42 @@ func TestAccKubernetesCluster_networkDataPlane(t *testing.T) { }) } +func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.apiServerInManagedSubnet(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + +func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.apiServerInBYOSubnet(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccKubernetesCluster_clusterPoolNodePublicIPTags(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -1037,6 +1073,129 @@ func TestAccKubernetesCluster_clusterPoolNetworkProfileUpdate(t *testing.T) { }) } +func (KubernetesClusterResource) apiServerInBYOSubnet(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} +resource "azurerm_virtual_network" "test" { + name = "acctestvirtnet%d" + address_space = ["10.0.0.0/8"] + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name +} +resource "azurerm_subnet" "test" { + name = "acctestsubnet%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.1.0.0/16"] + delegation { + name = "aks-delegation" + service_delegation { + actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] + name = "Microsoft.ContainerService/managedClusters" + } + } +} +resource "azurerm_subnet" "test1" { + name = "acctestsubnet1%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["10.2.0.0/16"] +} +resource "azurerm_user_assigned_identity" "test" { + name = "acctestRG-aks-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location +} +resource "azurerm_role_assignment" "test" { + scope = azurerm_subnet.test.id + role_definition_name = "Network Contributor" + principal_id = azurerm_user_assigned_identity.test.principal_id +} +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + api_server_access_profile { + vnet_integration_enabled = true + subnet_id = azurerm_subnet.test.id + } + linux_profile { + admin_username = "acctestuser%d" + ssh_key { + key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" + } + } + default_node_pool { + name = "default" + node_count = 2 + vm_size = "Standard_DS2_v2" + vnet_subnet_id = azurerm_subnet.test1.id + upgrade_settings { + max_surge = "10%%" + } + } + identity { + type = "UserAssigned" + identity_ids = [azurerm_user_assigned_identity.test.id] + } + network_profile { + network_plugin = "azure" + } + depends_on = [ + azurerm_role_assignment.test, + ] +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger) +} + +func (KubernetesClusterResource) apiServerInManagedSubnet(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + api_server_access_profile { + vnet_integration_enabled = true + } + linux_profile { + admin_username = "acctestuser%d" + ssh_key { + key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" + } + } + default_node_pool { + name = "default" + node_count = 2 + vm_size = "Standard_DS2_v2" + upgrade_settings { + max_surge = "10%%" + } + } + identity { + type = "SystemAssigned" + } + network_profile { + network_plugin = "azure" + } +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, data.RandomInteger) +} + func (KubernetesClusterResource) advancedNetworkingConfig(data acceptance.TestData, networkPlugin string) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go b/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go index cfcc79926375..7346ecabce01 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource_test.go @@ -941,6 +941,26 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test") r := KubernetesClusterNodePoolResource{} + if !features.FourPointOhBeta() { + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.workloadRuntime(data, "OCIContainer"), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.workloadRuntime(data, "KataMshvVmIsolation"), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) + return + } + data.ResourceTest(t, r, []acceptance.TestStep{ { Config: r.workloadRuntime(data, "OCIContainer"), @@ -952,6 +972,31 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) { }) } +func TestAccKubernetesClusterNodePool_customCATrustEnabled(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test") + r := KubernetesClusterNodePoolResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.customCATrustEnabled(data, true), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.customCATrustEnabled(data, false), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccKubernetesClusterNodePool_windowsProfileOutboundNatEnabled(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test") r := KubernetesClusterNodePoolResource{} @@ -2393,6 +2438,25 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" { } func (r KubernetesClusterNodePoolResource) other(data acceptance.TestData) string { + if !features.FourPointOhBeta() { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +%s + +resource "azurerm_kubernetes_cluster_node_pool" "test" { + name = "internal" + kubernetes_cluster_id = azurerm_kubernetes_cluster.test.id + vm_size = "Standard_DS2_v2" + node_count = 3 + fips_enabled = true + kubelet_disk_type = "OS" + message_of_the_day = "daily message" +} +`, r.templateConfig(data)) + } return fmt.Sprintf(` provider "azurerm" { features {} @@ -2646,6 +2710,41 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, workloadRuntime) } +func (KubernetesClusterNodePoolResource) customCATrustEnabled(data acceptance.TestData, enabled bool) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_D2s_v3" + upgrade_settings { + max_surge = "10%%" + } + } + identity { + type = "SystemAssigned" + } +} +resource "azurerm_kubernetes_cluster_node_pool" "test" { + name = "internal" + kubernetes_cluster_id = azurerm_kubernetes_cluster.test.id + vm_size = "Standard_D2s_v3" + custom_ca_trust_enabled = "%t" +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled) +} + func (KubernetesClusterNodePoolResource) windowsProfileOutboundNatEnabled(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_other_resource_test.go b/internal/services/containers/kubernetes_cluster_other_resource_test.go index a98144d837ce..e93ea0060203 100644 --- a/internal/services/containers/kubernetes_cluster_other_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_other_resource_test.go @@ -270,6 +270,24 @@ func TestAccKubernetesCluster_nodePoolOther(t *testing.T) { }) } +func TestAccKubernetesCluster_nodePoolKataMshvVmIsolation(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.nodePoolKataMshvVmIsolation(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccKubernetesCluster_upgradeSkuTier(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -898,6 +916,31 @@ func TestAccKubernetesCluster_workloadIdentity(t *testing.T) { }) } +func TestAccKubernetesCluster_customCATrustEnabled(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.customCATrustEnabled(data, true), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.customCATrustEnabled(data, false), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccKubernetesCluster_webAppRoutingWithMultipleDnsZone(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -1049,6 +1092,44 @@ func TestAccKubernetesCluster_nodeOsUpgradeChannel(t *testing.T) { }) } +func TestAccKubernetesCluster_customCaTrustCerts(t *testing.T) { + if features.FourPointOhBeta() { + t.Skip("Skipping this test in 4.0 beta as it is not supported") + } + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + fakeCertList := []string{ + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzYKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURmakNDQW1hZ0F3SUJBZ0lFZnlWdk56QU5CZ2txaGtpRzl3MEJBUXNGQURCWU1Rc3dDUVlEVlFRR0V3SlEKVERFVU1CSUdBMVVFQXd3TFJtRnJaU0JEWlhKMElESXhGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFYwpNQm9HQTFVRUNnd1RSR1ZtWVhWc2RDQkRiMjF3WVc1NUlFeDBaREFlRncweU16QTJNRFF3TnpJME1qZGFGdzB5Ck5UQTJNRE13TnpJME1qZGFNRmd4Q3pBSkJnTlZCQVlUQWxCTU1SUXdFZ1lEVlFRRERBdEdZV3RsSUVObGNuUWcKTWpFVk1CTUdBMVVFQnd3TVJHVm1ZWFZzZENCRGFYUjVNUnd3R2dZRFZRUUtEQk5FWldaaGRXeDBJRU52YlhCaApibmtnVEhSa01JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBMENTdVdUaGNjSG5MCkhFdjk4SUVNc2JLY3h4YVh4YTZiRXl1Yy9sUjRackpVN2p6eVlWNGVscTV5WTgwdDFCM0MyV3E2SXFoajErSGYKYW0xaStsU1FTejM1eWNnTWlwSWp2cUxKOVIzMVF0Wi9TRURkdGV2b2JqbytEa1dCOE55cG9Ia0pVbEIyQnR6ZgpOK09KeVFSdXU1b1cya2c5OE5Bd3JuTGpmQ0lremVWcFh5d0l4Tkx2ZmFrVGxpNWpYdG9WWG5pOTU5bmtINWVwClkrRnVoSEQwaU5CS25XYVkxR2QwVGhhSHNwTERmNFUycmo2WE5SZHd6QVZoVkdhUm02cndvSHRZeDVrYys1ZWMKQ0F4UEdRWFRzTzJUTHVrQzJ2YXI0M3RUM0ZjSC9taDRST2JaaThZS2xSQ3Fldm1QU1RmZ293RUFkTjlvSmxyRApXN2lzN2NnQjhRSURBUUFCbzFBd1RqQWRCZ05WSFE0RUZnUVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293Ckh3WURWUjBqQkJnd0ZvQVVuRkRqN0pBQW9WZ2NzQkgyNzdMOHZlM0Q4U293REFZRFZSMFRCQVV3QXdFQi96QU4KQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBT0diT0Zyek4rN2YxbzhJSDNtMXZxT3IyTUtvNEZMWExGRjBVbEhkNApwZXRhL05aQjArUmQ3TnUrOCtnUnlUbEJWZU9EZjN5SXU0TlFCUU92MlNqdS9Jakd0MUtmaUF3WkUwT1RUQXc3CnhIWStsMVBJWEFFVWNqNk00cjFKQzc4ZVZrc2pycTZoV1RPZ0RrSVZuRjY3bXlReXduR25EY1k0d0Fqc2pUajgKKzR4NTIrRi9QaVNQVGtjUFNuN0s2UjQzaEt5QUs2Z0poOHE5cVNhME5RQ2U2czhwTGU2SVY5SElWVVFFVERVOQpsM1VWWHNBMGx4dlB0blU1TXo2QWQ5cDA5L2w4d3o0cUdBdGFCUEd3K0R2cTNlaHdTd2VZZ3VHSktDQjhjb01JCjJRVUo0Zi9mNkFNVWtMeWxYZ3RSUEt1QjA3d3YwTmk1eWI5MjlFY1FJQ0l2dFE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t", + } + + fakeCertList2 := []string{ + "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURjRENDQWxpZ0F3SUJBZ0lFU1QwSUhEQU5CZ2txaGtpRzl3MEJBUXNGQURCUk1Rc3dDUVlEVlFRR0V3SlEKVERFTk1Bc0dBMVVFQXd3RVZHVnpkREVWTUJNR0ExVUVCd3dNUkdWbVlYVnNkQ0JEYVhSNU1Sd3dHZ1lEVlFRSwpEQk5FWldaaGRXeDBJRU52YlhCaGJua2dUSFJrTUI0WERUSXpNRFV5T0RFeE1qY3dNMW9YRFRNek1EVXlOVEV4Ck1qY3dNMW93VVRFTE1Ba0dBMVVFQmhNQ1VFd3hEVEFMQmdOVkJBTU1CRlJsYzNReEZUQVRCZ05WQkFjTURFUmwKWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFQ2d3VFJHVm1ZWFZzZENCRGIyMXdZVzU1SUV4MFpEQ0NBU0l3RFFZSgpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0NBUW9DZ2dFQkFLN2JIYWtxSkdRMWVBOUFHUmlhNGl2anNDRXlGMDhDCjNpSzJZeWthNkREeldmTk1tRWpOUjJiQVZOMEhlLy9pWTd1VjJ2dXl6V1UxMzZGVkdMZkdyeTZGOHNQQUZaSzcKSE4vcWk1QVp6MUpoOGdWSTRwS1pjZEFxQS81clF3VVlvWVN3Q245dGVOYytsbU1ZUk5OcTVwdlV2NjcrNEM3MgpPc3BOSUxSclhBbWNUb1YveVRZVzFKWDBOeEJJSHZZaFZXUE9LQXpRZDQ5UEpSeFpqMUgydCszMEFsazgzTDFwClFzTGx2SzV3MjJpeXdkYVpRN1lmV0xXd1hPQzVPWXdRTUw1R3BHUFNQaEdxdjhqSUhpcHBVeTdrRDlNWFFZOFoKdDl2QkczMzVWSEdlUjI2QnNQQXRFbTJjR05ocjA5cmRvdWJGd2tDR05OYXNVamFoVW9CKzhPY0NBd0VBQWFOUQpNRTR3SFFZRFZSME9CQllFRk9CNmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1COEdBMVVkSXdRWU1CYUFGT0I2CmNpTGtUL21Cc2xXSm5Na2phQzZqbjd4ek1Bd0dBMVVkRXdRRk1BTUJBZjh3RFFZSktvWklodmNOQVFFTEJRQUQKZ2dFQkFKTklHdHJpeFlCRUc1Yy9iQWdOMHlMOEJvOW9nN29ha0hVMUc5TjBxOUNWWXhjOVhma2ZUaEhYOVBUeApMbVNGcHJEQlAyYnVGTzVIUDFpbnNFT1E2N1lGanAvRjVJWGdaQ2twZUpGdDBTL0R3N2ZRbFJJN2RCNGQzNmIzCmE1R2txU0M4aFlZemxLUm9DRGNhalp4QmdoVUFxK0tnTnV4RmNsM1Fnd1Uyam1QbkU4a1A4TmgyM3hlVUJ3WEkKL3pqbU1rdjV4SFhKdHBpdlpzTlpSSUttQW56RU9TWGlRK2JMTStTdlhtSkhYd29YYTZyTXg4YmkySzV4WkhIRwpkUHA1TnQ3L2dxOUdXcm95SkVjSFpEclBiSnR2WGFibTZYUXpxTTFYUzA3SDlaSFBXc0dENGlBM1k0T3JUUlRCClZ5blRPUDl5U3cwbklaVEk4YjZuR2RHTzBOOD0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==", + } + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.customCATrustCertificates(data, fakeCertList), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").Exists(), + ), + }, + data.ImportStep(), + { + Config: r.customCATrustCertificates(data, fakeCertList2), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.0").Exists(), + check.That(data.ResourceName).Key("custom_ca_trust_certificates_base64.1").DoesNotExist(), + ), + }, + data.ImportStep(), + }) +} + func TestAccKubernetesCluster_snapshotId(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -1785,6 +1866,42 @@ resource "azurerm_kubernetes_cluster" "test" { } func (KubernetesClusterResource) nodePoolOther(data acceptance.TestData) string { + if !features.FourPointOhBeta() { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + fips_enabled = true + kubelet_disk_type = "OS" + message_of_the_day = "daily message" + workload_runtime = "OCIContainer" + upgrade_settings { + max_surge = "10%%" + } + } + + identity { + type = "SystemAssigned" + } +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger) + } return fmt.Sprintf(` provider "azurerm" { features {} @@ -1820,6 +1937,42 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger) } +func (KubernetesClusterResource) nodePoolKataMshvVmIsolation(data acceptance.TestData) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_D2s_v3" + message_of_the_day = "daily message" + os_sku = "AzureLinux" + workload_runtime = "KataMshvVmIsolation" + upgrade_settings { + max_surge = "10%%" + } + } + + identity { + type = "SystemAssigned" + } +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger) +} + func (KubernetesClusterResource) skuConfigStandard(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { @@ -3277,6 +3430,36 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomString, data.RandomInteger, data.RandomInteger) } +func (KubernetesClusterResource) customCATrustEnabled(data acceptance.TestData, enabled bool) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_D2s_v3" + custom_ca_trust_enabled = "%t" + upgrade_settings { + max_surge = "10%%" + } + } + identity { + type = "SystemAssigned" + } +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, enabled) +} + func (KubernetesClusterResource) azureMonitorKubernetesMetricsEnabled(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { @@ -3443,6 +3626,44 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, nodeOsUpgradeChannel) } +func (KubernetesClusterResource) customCATrustCertificates(data acceptance.TestData, certsList []string) string { + + certsString := "" + + if certsList != nil { + certsString = "\"" + strings.Join(certsList, "\" ,\"") + "\"" + } + + return fmt.Sprintf(` +provider "azurerm" { + features {} +} +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + role_based_access_control_enabled = true + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + upgrade_settings { + max_surge = "10%%" + } + } + identity { + type = "SystemAssigned" + } + custom_ca_trust_certificates_base64 = [%s] +} +`, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, certsString) +} + func (KubernetesClusterResource) snapshotSource(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 6c24fb653a80..294f0182522f 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -15,6 +15,7 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/internal/acceptance" "github.com/hashicorp/terraform-provider-azurerm/internal/acceptance/check" "github.com/hashicorp/terraform-provider-azurerm/internal/clients" + "github.com/hashicorp/terraform-provider-azurerm/internal/features" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/utils" ) @@ -851,6 +852,47 @@ resource "azurerm_kubernetes_cluster" "test" { } func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { + if !features.FourPointOhBeta() { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + kubernetes_version = %q + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + upgrade_settings { + max_surge = "10%%" + } + } + + identity { + type = "SystemAssigned" + } + + storage_profile { + blob_driver_enabled = true + disk_driver_enabled = true + disk_driver_version = "v1" + file_driver_enabled = false + snapshot_controller_enabled = false + } +} + `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion) + } return fmt.Sprintf(` provider "azurerm" { features {} From 11f3a285816afc728605d85ac7d97a27e43da388 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Thu, 1 Aug 2024 16:47:42 +0800 Subject: [PATCH 6/8] fix tests format --- ...ubernetes_cluster_network_resource_test.go | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/internal/services/containers/kubernetes_cluster_network_resource_test.go b/internal/services/containers/kubernetes_cluster_network_resource_test.go index 0a39d70a7103..2607ff697d10 100644 --- a/internal/services/containers/kubernetes_cluster_network_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_network_resource_test.go @@ -1078,60 +1078,73 @@ func (KubernetesClusterResource) apiServerInBYOSubnet(data acceptance.TestData) provider "azurerm" { features {} } + resource "azurerm_resource_group" "test" { name = "acctestRG-aks-%d" location = "%s" } + resource "azurerm_virtual_network" "test" { name = "acctestvirtnet%d" address_space = ["10.0.0.0/8"] location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name } + resource "azurerm_subnet" "test" { name = "acctestsubnet%d" resource_group_name = azurerm_resource_group.test.name virtual_network_name = azurerm_virtual_network.test.name address_prefixes = ["10.1.0.0/16"] + delegation { name = "aks-delegation" + service_delegation { actions = ["Microsoft.Network/virtualNetworks/subnets/join/action"] name = "Microsoft.ContainerService/managedClusters" } } } + resource "azurerm_subnet" "test1" { name = "acctestsubnet1%d" resource_group_name = azurerm_resource_group.test.name virtual_network_name = azurerm_virtual_network.test.name address_prefixes = ["10.2.0.0/16"] } + resource "azurerm_user_assigned_identity" "test" { name = "acctestRG-aks-%d" resource_group_name = azurerm_resource_group.test.name location = azurerm_resource_group.test.location } + resource "azurerm_role_assignment" "test" { scope = azurerm_subnet.test.id role_definition_name = "Network Contributor" principal_id = azurerm_user_assigned_identity.test.principal_id } + resource "azurerm_kubernetes_cluster" "test" { name = "acctestaks%d" location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name dns_prefix = "acctestaks%d" + api_server_access_profile { vnet_integration_enabled = true subnet_id = azurerm_subnet.test.id } + linux_profile { admin_username = "acctestuser%d" + ssh_key { key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" } } + default_node_pool { name = "default" node_count = 2 @@ -1141,13 +1154,16 @@ resource "azurerm_kubernetes_cluster" "test" { max_surge = "10%%" } } + identity { type = "UserAssigned" identity_ids = [azurerm_user_assigned_identity.test.id] } + network_profile { network_plugin = "azure" } + depends_on = [ azurerm_role_assignment.test, ] @@ -1160,24 +1176,30 @@ func (KubernetesClusterResource) apiServerInManagedSubnet(data acceptance.TestDa provider "azurerm" { features {} } + resource "azurerm_resource_group" "test" { name = "acctestRG-aks-%d" location = "%s" } + resource "azurerm_kubernetes_cluster" "test" { name = "acctestaks%d" location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name dns_prefix = "acctestaks%d" + api_server_access_profile { vnet_integration_enabled = true } + linux_profile { admin_username = "acctestuser%d" + ssh_key { key_data = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCqaZoyiz1qbdOQ8xEf6uEu1cCwYowo5FHtsBhqLoDnnp7KUTEBN+L2NxRIfQ781rxV6Iq5jSav6b2Q8z5KiseOlvKA/RF2wqU0UPYqQviQhLmW6THTpmrv/YkUCuzxDpsH7DUDhZcwySLKVVe0Qm3+5N2Ta6UYH3lsDf9R9wTP2K/+vAnflKebuypNlmocIvakFWoZda18FOmsOoIVXQ8HWFNCuw9ZCunMSN62QGamCe3dL5cXlkgHYv7ekJE15IA9aOJcM7e90oeTqo+7HTcWfdu0qQqPWY5ujyMw/llas8tsXY85LFqRnr3gJ02bAscjc477+X+j/gkpFoN1QEmt terraform@demo.tld" } } + default_node_pool { name = "default" node_count = 2 @@ -1186,9 +1208,11 @@ resource "azurerm_kubernetes_cluster" "test" { max_surge = "10%%" } } + identity { type = "SystemAssigned" } + network_profile { network_plugin = "azure" } From 7938c006849dd221f9e5cd622ab51f602f0089e2 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 13 Aug 2024 14:22:15 +0800 Subject: [PATCH 7/8] update the deprecate messages --- .../services/containers/kubernetes_cluster_data_source.go | 4 ++-- .../containers/kubernetes_cluster_node_pool_resource.go | 4 ++-- .../services/containers/kubernetes_cluster_resource.go | 8 ++++---- internal/services/containers/kubernetes_nodepool.go | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 13277a9e8f9b..f4aad1c4d2a7 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -734,13 +734,13 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { Deprecated: "This property is deprecated and will be removed in v4.0 of the AzureRM Provider in favour of the `node_public_ip_enabled` property.", } resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeString, Computed: true, } resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeList, Computed: true, Elem: &pluginsdk.Schema{ diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource.go b/internal/services/containers/kubernetes_cluster_node_pool_resource.go index 4cf714ad072e..0383ab9bc00e 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource.go @@ -403,7 +403,7 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { if !features.FourPointOhBeta() { s["message_of_the_day"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeString, Optional: true, ForceNew: true, @@ -411,7 +411,7 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema { } s["custom_ca_trust_enabled"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeBool, Optional: true, } diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 56bcfd1ba3cf..ddbf84a8b64c 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -1505,18 +1505,18 @@ func resourceKubernetesCluster() *pluginsdk.Resource { ConflictsWith: []string{"api_server_access_profile.0.authorized_ip_ranges"}, } resource.Schema["api_server_access_profile"].Elem.(*pluginsdk.Resource).Schema["vnet_integration_enabled"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeBool, Optional: true, } resource.Schema["api_server_access_profile"].Elem.(*pluginsdk.Resource).Schema["subnet_id"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeString, Optional: true, ValidateFunc: commonids.ValidateSubnetID, } resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeList, Optional: true, MaxItems: 10, @@ -1526,7 +1526,7 @@ func resourceKubernetesCluster() *pluginsdk.Resource { }, } resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeString, Optional: true, Default: "v1", diff --git a/internal/services/containers/kubernetes_nodepool.go b/internal/services/containers/kubernetes_nodepool.go index 3d7c7704d419..038374d35da7 100644 --- a/internal/services/containers/kubernetes_nodepool.go +++ b/internal/services/containers/kubernetes_nodepool.go @@ -279,19 +279,19 @@ func SchemaDefaultNodePool() *pluginsdk.Schema { if !features.FourPointOhBeta() { s["custom_ca_trust_enabled"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeBool, Optional: true, } s["message_of_the_day"] = &pluginsdk.Schema{ - Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.", + Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.", Type: pluginsdk.TypeString, Optional: true, ForceNew: true, ValidateFunc: validation.StringIsNotEmpty, } - + s["type"].ValidateFunc = validation.StringInSlice([]string{ string(managedclusters.AgentPoolTypeAvailabilitySet), string(managedclusters.AgentPoolTypeVirtualMachineScaleSets), From bbaba186dd705bf8712de5d7d27a34f1da388f11 Mon Sep 17 00:00:00 2001 From: Heng Lu Date: Tue, 13 Aug 2024 15:06:00 +0800 Subject: [PATCH 8/8] fix panic --- .../kubernetes_cluster_node_pool_resource.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_node_pool_resource.go b/internal/services/containers/kubernetes_cluster_node_pool_resource.go index 0383ab9bc00e..c37594fcef88 100644 --- a/internal/services/containers/kubernetes_cluster_node_pool_resource.go +++ b/internal/services/containers/kubernetes_cluster_node_pool_resource.go @@ -983,15 +983,21 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter if model := resp.Model; model != nil && model.Properties != nil { props := model.Properties d.Set("zones", zones.FlattenUntyped(props.AvailabilityZones)) - if features.FourPointOh() { + + switch { + case features.FourPointOh(): d.Set("auto_scaling_enabled", props.EnableAutoScaling) d.Set("node_public_ip_enabled", props.EnableNodePublicIP) d.Set("host_encryption_enabled", props.EnableEncryptionAtHost) - } else { + case features.FourPointOhBeta(): d.Set("enable_auto_scaling", props.EnableAutoScaling) d.Set("enable_node_public_ip", props.EnableNodePublicIP) d.Set("enable_host_encryption", props.EnableEncryptionAtHost) + default: d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust) + d.Set("enable_auto_scaling", props.EnableAutoScaling) + d.Set("enable_node_public_ip", props.EnableNodePublicIP) + d.Set("enable_host_encryption", props.EnableEncryptionAtHost) } d.Set("fips_enabled", props.EnableFIPS) d.Set("ultra_ssd_enabled", props.EnableUltraSSD)