azurerm_kubernetes_cluster, azurerm_kubernetes_cluster_node_pool - deprecate preview features

internal/services/containers/kubernetes_cluster_data_source.go

@@ -324,14 +324,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
 				},
 			},
 
-			"custom_ca_trust_certificates_base64": {
-				Type:     pluginsdk.TypeList,
-				Computed: true,
-				Elem: &pluginsdk.Schema{
-					Type: pluginsdk.TypeString,
-				},
-			},
-
 			"oms_agent": {
 				Type:     pluginsdk.TypeList,
 				Computed: true,
@@ -654,10 +646,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
 							Type:     pluginsdk.TypeBool,
 							Computed: true,
 						},
-						"disk_driver_version": {
-							Type:     pluginsdk.TypeString,
-							Computed: true,
-						},
 						"file_driver_enabled": {
 							Type:     pluginsdk.TypeBool,
 							Computed: true,
@@ -745,6 +733,21 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
 			Computed:   true,
 			Deprecated: "This property is deprecated and will be removed in v4.0 of the AzureRM Provider in favour of the `node_public_ip_enabled` property.",
 		}
+		resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{
+			Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.",
+			Type:       pluginsdk.TypeString,
+			Computed:   true,
+		}
+
+		resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{
+			Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.",
+			Type:       pluginsdk.TypeList,
+			Computed:   true,
+			Elem: &pluginsdk.Schema{
+				Type: pluginsdk.TypeString,
+			},
+		}
+
 		resource.Schema["azure_active_directory_role_based_access_control"] = &pluginsdk.Schema{
 			Type:     pluginsdk.TypeList,
 			Computed: true,
@@ -865,9 +868,11 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}
 				return fmt.Errorf("setting `key_management_service`: %+v", err)
 			}
 
-			customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile)
-			if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil {
-				return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err)
+			if !features.FourPointOhBeta() {
+				customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile)
+				if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil {
+					return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err)
+				}
 			}
 
 			serviceMeshProfile := flattenKubernetesClusterAzureServiceMeshProfile(props.ServiceMeshProfile)
@@ -1018,11 +1023,6 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man
 			diskEnabled = *input.DiskCSIDriver.Enabled
 		}
 
-		diskVersion := ""
-		if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil {
-			diskVersion = *input.DiskCSIDriver.Version
-		}
-
 		fileEnabled := true
 		if input.FileCSIDriver != nil && input.FileCSIDriver.Enabled != nil {
 			fileEnabled = *input.FileCSIDriver.Enabled
@@ -1033,13 +1033,26 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man
 			snapshotController = *input.SnapshotController.Enabled
 		}
 
-		storageProfile = append(storageProfile, map[string]interface{}{
-			"blob_driver_enabled":         blobEnabled,
-			"disk_driver_enabled":         diskEnabled,
-			"disk_driver_version":         diskVersion,
-			"file_driver_enabled":         fileEnabled,
-			"snapshot_controller_enabled": snapshotController,
-		})
+		if !features.FourPointOhBeta() {
+			diskVersion := ""
+			if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil {
+				diskVersion = *input.DiskCSIDriver.Version
+			}
+			storageProfile = append(storageProfile, map[string]interface{}{
+				"blob_driver_enabled":         blobEnabled,
+				"disk_driver_enabled":         diskEnabled,
+				"disk_driver_version":         diskVersion,
+				"file_driver_enabled":         fileEnabled,
+				"snapshot_controller_enabled": snapshotController,
+			})
+		} else {
+			storageProfile = append(storageProfile, map[string]interface{}{
+				"blob_driver_enabled":         blobEnabled,
+				"disk_driver_enabled":         diskEnabled,
+				"file_driver_enabled":         fileEnabled,
+				"snapshot_controller_enabled": snapshotController,
+			})
+		}
 	}
 
 	return storageProfile

internal/services/containers/kubernetes_cluster_data_source_test.go

@@ -555,6 +555,9 @@ func TestAccDataSourceKubernetesCluster_microsoftDefender(t *testing.T) {
 }
 
 func TestAccDataSourceKubernetesCluster_customCaTrustCerts(t *testing.T) {
+	if features.FourPointOhBeta() {
+		t.Skip("Skipping this test in 4.0 beta as it is not supported")
+	}
 	data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test")
 	r := KubernetesClusterDataSource{}
 

internal/services/containers/kubernetes_cluster_network_resource_test.go

@@ -979,6 +979,9 @@ func TestAccKubernetesCluster_networkDataPlane(t *testing.T) {
 }
 
 func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) {
+	if features.FourPointOhBeta() {
+		t.Skip("Skipping this test in 4.0 beta as it is not supported")
+	}
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
 	r := KubernetesClusterResource{}
 
@@ -994,6 +997,9 @@ func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) {
 }
 
 func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) {
+	if features.FourPointOhBeta() {
+		t.Skip("Skipping this test in 4.0 beta as it is not supported")
+	}
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
 	r := KubernetesClusterResource{}
 

internal/services/containers/kubernetes_cluster_node_pool_resource.go

@@ -134,11 +134,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
 			ValidateFunc: capacityreservationgroups.ValidateCapacityReservationGroupID,
 		},
 
-		"custom_ca_trust_enabled": {
-			Type:     pluginsdk.TypeBool,
-			Optional: true,
-		},
-
 		"eviction_policy": {
 			Type:     pluginsdk.TypeString,
 			Optional: true,
@@ -195,13 +190,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
 			ForceNew: true,
 		},
 
-		"message_of_the_day": {
-			Type:         pluginsdk.TypeString,
-			Optional:     true,
-			ForceNew:     true,
-			ValidateFunc: validation.StringIsNotEmpty,
-		},
-
 		"mode": {
 			Type:     pluginsdk.TypeString,
 			Optional: true,
@@ -390,7 +378,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
 			ValidateFunc: validation.StringInSlice([]string{
 				string(agentpools.WorkloadRuntimeOCIContainer),
 				string(agentpools.WorkloadRuntimeWasmWasi),
-				string(agentpools.WorkloadRuntimeKataMshvVMIsolation),
 			}, false),
 		},
 
@@ -415,6 +402,20 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
 	}
 
 	if !features.FourPointOhBeta() {
+		s["message_of_the_day"] = &pluginsdk.Schema{
+			Deprecated:   "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.",
+			Type:         pluginsdk.TypeString,
+			Optional:     true,
+			ForceNew:     true,
+			ValidateFunc: validation.StringIsNotEmpty,
+		}
+
+		s["custom_ca_trust_enabled"] = &pluginsdk.Schema{
+			Deprecated: "This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details.",
+			Type:       pluginsdk.TypeBool,
+			Optional:   true,
+		}
+
 		s["os_sku"].ValidateFunc = validation.StringInSlice([]string{
 			string(agentpools.OSSKUAzureLinux),
 			string(agentpools.OSSKUCBLMariner),
@@ -423,6 +424,12 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
 			string(agentpools.OSSKUWindowsTwoZeroOneNine),
 			string(agentpools.OSSKUWindowsTwoZeroTwoTwo),
 		}, false)
+
+		s["workload_runtime"].ValidateFunc = validation.StringInSlice([]string{
+			string(agentpools.WorkloadRuntimeOCIContainer),
+			string(agentpools.WorkloadRuntimeWasmWasi),
+			string(agentpools.WorkloadRuntimeKataMshvVMIsolation),
+		}, false)
 	}
 
 	if !features.FourPointOh() {
@@ -546,7 +553,6 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
 	profile := agentpools.ManagedClusterAgentPoolProfileProperties{
 		OsType:                 pointer.To(agentpools.OSType(osType)),
 		EnableAutoScaling:      pointer.To(enableAutoScaling),
-		EnableCustomCATrust:    pointer.To(d.Get("custom_ca_trust_enabled").(bool)),
 		EnableFIPS:             pointer.To(d.Get("fips_enabled").(bool)),
 		EnableEncryptionAtHost: pointer.To(hostEncryption),
 		EnableUltraSSD:         pointer.To(d.Get("ultra_ssd_enabled").(bool)),
@@ -564,6 +570,10 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
 		Count: utils.Int64(int64(count)),
 	}
 
+	if !features.FourPointOhBeta() {
+		profile.EnableCustomCATrust = pointer.To(d.Get("custom_ca_trust_enabled").(bool))
+	}
+
 	if gpuInstanceProfile := d.Get("gpu_instance").(string); gpuInstanceProfile != "" {
 		profile.GpuInstanceProfile = pointer.To(agentpools.GPUInstanceProfile(gpuInstanceProfile))
 	}
@@ -625,12 +635,14 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
 		profile.NodeTaints = nodeTaints
 	}
 
-	if v := d.Get("message_of_the_day").(string); v != "" {
-		if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows {
-			return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)")
+	if !features.FourPointOhBeta() {
+		if v := d.Get("message_of_the_day").(string); v != "" {
+			if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows {
+				return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)")
+			}
+			messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v))
+			profile.MessageOfTheDay = &messageOfTheDayEncoded
 		}
-		messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v))
-		profile.MessageOfTheDay = &messageOfTheDayEncoded
 	}
 
 	if osDiskSizeGB := d.Get("os_disk_size_gb").(int); osDiskSizeGB > 0 {
@@ -810,8 +822,10 @@ func resourceKubernetesClusterNodePoolUpdate(d *pluginsdk.ResourceData, meta int
 		}
 	}
 
-	if d.HasChange("custom_ca_trust_enabled") {
-		props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool))
+	if !features.FourPointOhBeta() {
+		if d.HasChange("custom_ca_trust_enabled") {
+			props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool))
+		}
 	}
 
 	if d.HasChange("max_count") || enableAutoScaling {
@@ -969,16 +983,22 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter
 	if model := resp.Model; model != nil && model.Properties != nil {
 		props := model.Properties
 		d.Set("zones", zones.FlattenUntyped(props.AvailabilityZones))
-		if features.FourPointOh() {
+
+		switch {
+		case features.FourPointOh():
 			d.Set("auto_scaling_enabled", props.EnableAutoScaling)
 			d.Set("node_public_ip_enabled", props.EnableNodePublicIP)
 			d.Set("host_encryption_enabled", props.EnableEncryptionAtHost)
-		} else {
+		case features.FourPointOhBeta():
+			d.Set("enable_auto_scaling", props.EnableAutoScaling)
+			d.Set("enable_node_public_ip", props.EnableNodePublicIP)
+			d.Set("enable_host_encryption", props.EnableEncryptionAtHost)
+		default:
+			d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust)
 			d.Set("enable_auto_scaling", props.EnableAutoScaling)
 			d.Set("enable_node_public_ip", props.EnableNodePublicIP)
 			d.Set("enable_host_encryption", props.EnableEncryptionAtHost)
 		}
-		d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust)
 		d.Set("fips_enabled", props.EnableFIPS)
 		d.Set("ultra_ssd_enabled", props.EnableUltraSSD)
 
@@ -1028,15 +1048,17 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter
 		}
 		d.Set("max_count", maxCount)
 
-		messageOfTheDay := ""
-		if props.MessageOfTheDay != nil {
-			messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay)
-			if err != nil {
-				return fmt.Errorf("setting `message_of_the_day`: %+v", err)
+		if !features.FourPointOhBeta() {
+			messageOfTheDay := ""
+			if props.MessageOfTheDay != nil {
+				messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay)
+				if err != nil {
+					return fmt.Errorf("setting `message_of_the_day`: %+v", err)
+				}
+				messageOfTheDay = string(messageOfTheDayDecoded)
 			}
-			messageOfTheDay = string(messageOfTheDayDecoded)
+			d.Set("message_of_the_day", messageOfTheDay)
 		}
-		d.Set("message_of_the_day", messageOfTheDay)
 
 		maxPods := 0
 		if props.MaxPods != nil {

internal/services/containers/kubernetes_cluster_node_pool_resource_test.go

@@ -944,6 +944,26 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) {
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test")
 	r := KubernetesClusterNodePoolResource{}
 
+	if !features.FourPointOhBeta() {
+		data.ResourceTest(t, r, []acceptance.TestStep{
+			{
+				Config: r.workloadRuntime(data, "OCIContainer"),
+				Check: acceptance.ComposeTestCheckFunc(
+					check.That(data.ResourceName).ExistsInAzure(r),
+				),
+			},
+			data.ImportStep(),
+			{
+				Config: r.workloadRuntime(data, "KataMshvVmIsolation"),
+				Check: acceptance.ComposeTestCheckFunc(
+					check.That(data.ResourceName).ExistsInAzure(r),
+				),
+			},
+			data.ImportStep(),
+		})
+		return
+	}
+
 	data.ResourceTest(t, r, []acceptance.TestStep{
 		{
 			Config: r.workloadRuntime(data, "OCIContainer"),
@@ -952,17 +972,13 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) {
 			),
 		},
 		data.ImportStep(),
-		{
-			Config: r.workloadRuntime(data, "KataMshvVmIsolation"),
-			Check: acceptance.ComposeTestCheckFunc(
-				check.That(data.ResourceName).ExistsInAzure(r),
-			),
-		},
-		data.ImportStep(),
 	})
 }
 
 func TestAccKubernetesClusterNodePool_customCATrustEnabled(t *testing.T) {
+	if features.FourPointOhBeta() {
+		t.Skip("Skipping this test in 4.0 beta as it is not supported")
+	}
 	data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test")
 	r := KubernetesClusterNodePoolResource{}
 
@@ -2425,7 +2441,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" {
 }
 
 func (r KubernetesClusterNodePoolResource) other(data acceptance.TestData) string {
-	return fmt.Sprintf(`
+	if !features.FourPointOhBeta() {
+		return fmt.Sprintf(`
 provider "azurerm" {
   features {}
 }
@@ -2441,6 +2458,23 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" {
   kubelet_disk_type     = "OS"
   message_of_the_day    = "daily message"
 }
+`, r.templateConfig(data))
+	}
+	return fmt.Sprintf(`
+provider "azurerm" {
+  features {}
+}
+
+%s
+
+resource "azurerm_kubernetes_cluster_node_pool" "test" {
+  name                  = "internal"
+  kubernetes_cluster_id = azurerm_kubernetes_cluster.test.id
+  vm_size               = "Standard_DS2_v2"
+  node_count            = 3
+  fips_enabled          = true
+  kubelet_disk_type     = "OS"
+}
 `, r.templateConfig(data))
 }