Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

azurerm_kubernetes_cluster, azurerm_kubernetes_cluster_node_pool - deprecate preview features #26863

Merged
merged 9 commits into from
Aug 15, 2024
Merged

azurerm_kubernetes_cluster, azurerm_kubernetes_cluster_node_pool - deprecate preview features #26863

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b7b3833
`azurerm_kubernetes_cluster`, `azurerm_kubernetes_cluster_node_pool` …
ms-henglu Jul 30, 2024
30998ff
make terrafmt
ms-henglu Jul 30, 2024
5a2fa72
only set custom_ca_trust_enabled if it's not 4.0 for node pool
ms-henglu Jul 30, 2024
a08406c
remove tests
ms-henglu Jul 31, 2024
4fd4f6b
add skip on the tests
ms-henglu Aug 1, 2024
11f3a28
fix tests format
ms-henglu Aug 1, 2024
8940da2
Merge branch 'main' into ticket-27055303-aks-switch-sdk-2024-05-01
ms-henglu Aug 13, 2024
7938c00
update the deprecate messages
ms-henglu Aug 13, 2024
bbaba18
fix panic
ms-henglu Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 40 additions & 27 deletions internal/services/containers/kubernetes_cluster_data_source.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,14 +324,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
},
},

"custom_ca_trust_certificates_base64": {
Type: pluginsdk.TypeList,
Computed: true,
Elem: &pluginsdk.Schema{
Type: pluginsdk.TypeString,
},
},

"oms_agent": {
Type: pluginsdk.TypeList,
Computed: true,
Expand Down Expand Up @@ -654,10 +646,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
Type: pluginsdk.TypeBool,
Computed: true,
},
"disk_driver_version": {
Type: pluginsdk.TypeString,
Computed: true,
},
"file_driver_enabled": {
Type: pluginsdk.TypeBool,
Computed: true,
Expand Down Expand Up @@ -745,6 +733,21 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource {
Computed: true,
Deprecated: "This property is deprecated and will be removed in v4.0 of the AzureRM Provider in favour of the `node_public_ip_enabled` property.",
}
resource.Schema["storage_profile"].Elem.(*pluginsdk.Resource).Schema["disk_driver_version"] = &pluginsdk.Schema{
Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please update all of the deprecation messages to the following:
This property is not available in the stable API and will be removed in v4.0 of the Azure Provider. Please see https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/guides/4.0-upgrade-guide#aks-migration-to-stable-api for more details

The link in the message isn't accessible right now, but it will be when the release goes out this week.

Type: pluginsdk.TypeString,
Computed: true,
}

resource.Schema["custom_ca_trust_certificates_base64"] = &pluginsdk.Schema{
Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.",
Type: pluginsdk.TypeList,
Computed: true,
Elem: &pluginsdk.Schema{
Type: pluginsdk.TypeString,
},
}

resource.Schema["azure_active_directory_role_based_access_control"] = &pluginsdk.Schema{
Type: pluginsdk.TypeList,
Computed: true,
Expand Down Expand Up @@ -865,9 +868,11 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}
return fmt.Errorf("setting `key_management_service`: %+v", err)
}

customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile)
if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil {
return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err)
if !features.FourPointOhBeta() {
customCaTrustCertList := flattenCustomCaTrustCerts(props.SecurityProfile)
if err := d.Set("custom_ca_trust_certificates_base64", customCaTrustCertList); err != nil {
return fmt.Errorf("setting `custom_ca_trust_certificates_base64`: %+v", err)
}
}

serviceMeshProfile := flattenKubernetesClusterAzureServiceMeshProfile(props.ServiceMeshProfile)
Expand Down Expand Up @@ -1018,11 +1023,6 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man
diskEnabled = *input.DiskCSIDriver.Enabled
}

diskVersion := ""
if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil {
diskVersion = *input.DiskCSIDriver.Version
}

fileEnabled := true
if input.FileCSIDriver != nil && input.FileCSIDriver.Enabled != nil {
fileEnabled = *input.FileCSIDriver.Enabled
Expand All @@ -1033,13 +1033,26 @@ func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.Man
snapshotController = *input.SnapshotController.Enabled
}

storageProfile = append(storageProfile, map[string]interface{}{
"blob_driver_enabled": blobEnabled,
"disk_driver_enabled": diskEnabled,
"disk_driver_version": diskVersion,
"file_driver_enabled": fileEnabled,
"snapshot_controller_enabled": snapshotController,
})
if !features.FourPointOhBeta() {
diskVersion := ""
if input.DiskCSIDriver != nil && input.DiskCSIDriver.Version != nil {
diskVersion = *input.DiskCSIDriver.Version
}
storageProfile = append(storageProfile, map[string]interface{}{
"blob_driver_enabled": blobEnabled,
"disk_driver_enabled": diskEnabled,
"disk_driver_version": diskVersion,
"file_driver_enabled": fileEnabled,
"snapshot_controller_enabled": snapshotController,
})
} else {
storageProfile = append(storageProfile, map[string]interface{}{
"blob_driver_enabled": blobEnabled,
"disk_driver_enabled": diskEnabled,
"file_driver_enabled": fileEnabled,
"snapshot_controller_enabled": snapshotController,
})
}
}

return storageProfile
Expand Down
3 changes: 3 additions & 0 deletions internal/services/containers/kubernetes_cluster_data_source_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,9 @@ func TestAccDataSourceKubernetesCluster_microsoftDefender(t *testing.T) {
}

func TestAccDataSourceKubernetesCluster_customCaTrustCerts(t *testing.T) {
Copy link
Member

@stephybun stephybun Aug 1, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We shouldn't be removing these tests until after 4.0

  • We still need the ability to run these in case there are severe issues with any features that are still available in 3.x
  • In case of unforeseen consequences that may require us to roll back anything

Can you please skip the tests using the flag for now. We can worry about cleaning up unused tests and configs post major release.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it! I've updated this PR as suggested.

if features.FourPointOhBeta() {
t.Skip("Skipping this test in 4.0 beta as it is not supported")
}
data := acceptance.BuildTestData(t, "data.azurerm_kubernetes_cluster", "test")
r := KubernetesClusterDataSource{}

Expand Down
6 changes: 6 additions & 0 deletions internal/services/containers/kubernetes_cluster_network_resource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,9 @@ func TestAccKubernetesCluster_networkDataPlane(t *testing.T) {
}

func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) {
if features.FourPointOhBeta() {
t.Skip("Skipping this test in 4.0 beta as it is not supported")
}
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
r := KubernetesClusterResource{}

Expand All @@ -994,6 +997,9 @@ func TestAccKubernetesCluster_apiServerInManagedSubnet(t *testing.T) {
}

func TestAccKubernetesCluster_apiServerInBYOSubnet(t *testing.T) {
if features.FourPointOhBeta() {
t.Skip("Skipping this test in 4.0 beta as it is not supported")
}
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test")
r := KubernetesClusterResource{}

Expand Down
74 changes: 45 additions & 29 deletions internal/services/containers/kubernetes_cluster_node_pool_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
ValidateFunc: capacityreservationgroups.ValidateCapacityReservationGroupID,
},

"custom_ca_trust_enabled": {
Type: pluginsdk.TypeBool,
Optional: true,
},

"eviction_policy": {
Type: pluginsdk.TypeString,
Optional: true,
Expand Down Expand Up @@ -195,13 +190,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
ForceNew: true,
},

"message_of_the_day": {
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ValidateFunc: validation.StringIsNotEmpty,
},

"mode": {
Type: pluginsdk.TypeString,
Optional: true,
Expand Down Expand Up @@ -390,7 +378,6 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
ValidateFunc: validation.StringInSlice([]string{
string(agentpools.WorkloadRuntimeOCIContainer),
string(agentpools.WorkloadRuntimeWasmWasi),
string(agentpools.WorkloadRuntimeKataMshvVMIsolation),
}, false),
},

Expand All @@ -415,6 +402,20 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
}

if !features.FourPointOhBeta() {
s["message_of_the_day"] = &pluginsdk.Schema{
Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.",
Type: pluginsdk.TypeString,
Optional: true,
ForceNew: true,
ValidateFunc: validation.StringIsNotEmpty,
}

s["custom_ca_trust_enabled"] = &pluginsdk.Schema{
Deprecated: "This feature is a preview feature and will be removed in version 4.0 of the AzureRM Provider.",
Type: pluginsdk.TypeBool,
Optional: true,
}

s["os_sku"].ValidateFunc = validation.StringInSlice([]string{
string(agentpools.OSSKUAzureLinux),
string(agentpools.OSSKUCBLMariner),
Expand All @@ -423,6 +424,12 @@ func resourceKubernetesClusterNodePoolSchema() map[string]*pluginsdk.Schema {
string(agentpools.OSSKUWindowsTwoZeroOneNine),
string(agentpools.OSSKUWindowsTwoZeroTwoTwo),
}, false)

s["workload_runtime"].ValidateFunc = validation.StringInSlice([]string{
string(agentpools.WorkloadRuntimeOCIContainer),
string(agentpools.WorkloadRuntimeWasmWasi),
string(agentpools.WorkloadRuntimeKataMshvVMIsolation),
}, false)
}

if !features.FourPointOh() {
Expand Down Expand Up @@ -546,7 +553,6 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
profile := agentpools.ManagedClusterAgentPoolProfileProperties{
OsType: pointer.To(agentpools.OSType(osType)),
EnableAutoScaling: pointer.To(enableAutoScaling),
EnableCustomCATrust: pointer.To(d.Get("custom_ca_trust_enabled").(bool)),
EnableFIPS: pointer.To(d.Get("fips_enabled").(bool)),
EnableEncryptionAtHost: pointer.To(hostEncryption),
EnableUltraSSD: pointer.To(d.Get("ultra_ssd_enabled").(bool)),
Expand All @@ -564,6 +570,10 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
Count: utils.Int64(int64(count)),
}

if !features.FourPointOhBeta() {
profile.EnableCustomCATrust = pointer.To(d.Get("custom_ca_trust_enabled").(bool))
}

if gpuInstanceProfile := d.Get("gpu_instance").(string); gpuInstanceProfile != "" {
profile.GpuInstanceProfile = pointer.To(agentpools.GPUInstanceProfile(gpuInstanceProfile))
}
Expand Down Expand Up @@ -625,12 +635,14 @@ func resourceKubernetesClusterNodePoolCreate(d *pluginsdk.ResourceData, meta int
profile.NodeTaints = nodeTaints
}

if v := d.Get("message_of_the_day").(string); v != "" {
if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows {
return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)")
if !features.FourPointOhBeta() {
if v := d.Get("message_of_the_day").(string); v != "" {
if profile.OsType != nil && *profile.OsType == agentpools.OSTypeWindows {
return fmt.Errorf("`message_of_the_day` cannot be specified for Windows nodes and must be a static string (i.e. will be printed raw and not executed as a script)")
}
messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v))
profile.MessageOfTheDay = &messageOfTheDayEncoded
}
messageOfTheDayEncoded := base64.StdEncoding.EncodeToString([]byte(v))
profile.MessageOfTheDay = &messageOfTheDayEncoded
}

if osDiskSizeGB := d.Get("os_disk_size_gb").(int); osDiskSizeGB > 0 {
Expand Down Expand Up @@ -810,8 +822,10 @@ func resourceKubernetesClusterNodePoolUpdate(d *pluginsdk.ResourceData, meta int
}
}

if d.HasChange("custom_ca_trust_enabled") {
props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool))
if !features.FourPointOhBeta() {
if d.HasChange("custom_ca_trust_enabled") {
props.EnableCustomCATrust = utils.Bool(d.Get("custom_ca_trust_enabled").(bool))
}
}

if d.HasChange("max_count") || enableAutoScaling {
Expand Down Expand Up @@ -977,8 +991,8 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter
d.Set("enable_auto_scaling", props.EnableAutoScaling)
d.Set("enable_node_public_ip", props.EnableNodePublicIP)
d.Set("enable_host_encryption", props.EnableEncryptionAtHost)
d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust)
}
d.Set("custom_ca_trust_enabled", props.EnableCustomCATrust)
d.Set("fips_enabled", props.EnableFIPS)
d.Set("ultra_ssd_enabled", props.EnableUltraSSD)

Expand Down Expand Up @@ -1028,15 +1042,17 @@ func resourceKubernetesClusterNodePoolRead(d *pluginsdk.ResourceData, meta inter
}
d.Set("max_count", maxCount)

messageOfTheDay := ""
if props.MessageOfTheDay != nil {
messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay)
if err != nil {
return fmt.Errorf("setting `message_of_the_day`: %+v", err)
if !features.FourPointOhBeta() {
messageOfTheDay := ""
if props.MessageOfTheDay != nil {
messageOfTheDayDecoded, err := base64.StdEncoding.DecodeString(*props.MessageOfTheDay)
if err != nil {
return fmt.Errorf("setting `message_of_the_day`: %+v", err)
}
messageOfTheDay = string(messageOfTheDayDecoded)
}
messageOfTheDay = string(messageOfTheDayDecoded)
d.Set("message_of_the_day", messageOfTheDay)
}
d.Set("message_of_the_day", messageOfTheDay)

maxPods := 0
if props.MaxPods != nil {
Expand Down
50 changes: 42 additions & 8 deletions internal/services/containers/kubernetes_cluster_node_pool_resource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -941,6 +941,26 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test")
r := KubernetesClusterNodePoolResource{}

if !features.FourPointOhBeta() {
data.ResourceTest(t, r, []acceptance.TestStep{
{
Config: r.workloadRuntime(data, "OCIContainer"),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
{
Config: r.workloadRuntime(data, "KataMshvVmIsolation"),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
return
}

data.ResourceTest(t, r, []acceptance.TestStep{
{
Config: r.workloadRuntime(data, "OCIContainer"),
Expand All @@ -949,17 +969,13 @@ func TestAccKubernetesClusterNodePool_workloadRuntime(t *testing.T) {
),
},
data.ImportStep(),
{
Config: r.workloadRuntime(data, "KataMshvVmIsolation"),
Check: acceptance.ComposeTestCheckFunc(
check.That(data.ResourceName).ExistsInAzure(r),
),
},
data.ImportStep(),
})
}

func TestAccKubernetesClusterNodePool_customCATrustEnabled(t *testing.T) {
if features.FourPointOhBeta() {
t.Skip("Skipping this test in 4.0 beta as it is not supported")
}
data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster_node_pool", "test")
r := KubernetesClusterNodePoolResource{}

Expand Down Expand Up @@ -2422,7 +2438,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" {
}

func (r KubernetesClusterNodePoolResource) other(data acceptance.TestData) string {
return fmt.Sprintf(`
if !features.FourPointOhBeta() {
return fmt.Sprintf(`
provider "azurerm" {
features {}
}
Expand All @@ -2438,6 +2455,23 @@ resource "azurerm_kubernetes_cluster_node_pool" "test" {
kubelet_disk_type = "OS"
message_of_the_day = "daily message"
}
`, r.templateConfig(data))
}
return fmt.Sprintf(`
provider "azurerm" {
features {}
}

%s

resource "azurerm_kubernetes_cluster_node_pool" "test" {
name = "internal"
kubernetes_cluster_id = azurerm_kubernetes_cluster.test.id
vm_size = "Standard_DS2_v2"
node_count = 3
fips_enabled = true
kubelet_disk_type = "OS"
}
`, r.templateConfig(data))
}

Expand Down
Loading
Loading