From 7879d2d7837cbd89edd134ab7d0fe5ca750ae84f Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Tue, 3 Jan 2023 14:49:09 +0100 Subject: [PATCH 1/7] feat(aks): Ability to use KMS encryption --- .../kubernetes_cluster_data_source.go | 45 +++++++++++++ .../containers/kubernetes_cluster_resource.go | 67 +++++++++++++++++++ .../kubernetes_cluster_resource_test.go | 51 ++++++++++++++ .../docs/d/kubernetes_cluster.html.markdown | 14 ++++ .../docs/r/kubernetes_cluster.html.markdown | 15 +++++ 5 files changed, 192 insertions(+) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 88487f654159..30e0678a307b 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -380,6 +380,31 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { "identity": commonschema.SystemOrUserAssignedIdentityComputed(), + "key_vault_kms": { + Type: pluginsdk.TypeList, + Computed: true, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "enabled": { + Type: pluginsdk.TypeBool, + Computed: true, + }, + "key_id": { + Type: pluginsdk.TypeString, + Computed: true, + }, + "key_vault_network_access": { + Type: pluginsdk.TypeString, + Computed: true, + }, + "key_vault_resource_id": { + Type: pluginsdk.TypeString, + Computed: true, + }, + }, + }, + }, + "kubernetes_version": { Type: pluginsdk.TypeString, Computed: true, @@ -718,6 +743,11 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} return fmt.Errorf("setting `agent_pool_profile`: %+v", err) } + azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) + if err := d.Set("key_vault_kms", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_vault_kms`: %+v", err) + } + kubeletIdentity, err := flattenKubernetesClusterDataSourceIdentityProfile(props.IdentityProfile) if err != nil { return err @@ -827,6 +857,21 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} return nil } +func flattenKubernetesClusterDataSourceKeyVaultKms(input *managedclusters.AzureKeyVaultKms) []interface{} { + azureKeyVaultKms := make([]interface{}, 0) + + if input != nil { + azureKeyVaultKms = append(azureKeyVaultKms, map[string]interface{}{ + "enabled": input.Enabled, + "key_id": input.KeyId, + "key_vault_network_access": input.KeyVaultNetworkAccess, + "key_vault_resource_id": input.KeyVaultResourceId, + }) + } + + return azureKeyVaultKms +} + func flattenKubernetesClusterDataSourceStorageProfile(input *managedclusters.ManagedClusterStorageProfile) []interface{} { storageProfile := make([]interface{}, 0) diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index c5d19c7349f9..18909777fef5 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -31,6 +31,7 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/kubernetes" "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/migration" containerValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/validate" + keyVaultValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/validate" networkValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/network/validate" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/suppress" @@ -709,6 +710,37 @@ func resourceKubernetesCluster() *pluginsdk.Resource { }, }, + "key_vault_kms": { + Type: pluginsdk.TypeList, + Optional: true, + ForceNew: false, + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "enabled": { + Type: pluginsdk.TypeBool, + Required: true, + }, + "key_id": { + Type: pluginsdk.TypeString, + Required: true, + ValidateFunc: keyVaultValidate.NestedItemId, + }, + "key_vault_network_access": { + Type: pluginsdk.TypeString, + Default: string(managedclusters.KeyVaultNetworkAccessTypesPublic), + Optional: true, + ValidateFunc: validation.StringInSlice(managedclusters.PossibleValuesForKeyVaultNetworkAccessTypes(), false), + }, + "key_vault_resource_id": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: keyVaultValidate.VaultID, + }, + }, + }, + }, + "microsoft_defender": { Type: pluginsdk.TypeList, Optional: true, @@ -1363,6 +1395,9 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} IntervalHours: utils.Int64(int64(d.Get("image_cleaner_interval_hours").(int))), } + azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) + securityProfile.AzureKeyVaultKms = expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + parameters := managedclusters.ManagedCluster{ Name: utils.String(id.ResourceName), ExtendedLocation: expandEdgeZone(d.Get("edge_zone").(string)), @@ -1834,6 +1869,13 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.OidcIssuerProfile = oidcIssuerProfile } + if d.HasChanges("key_vault_kms") { + updateCluster = true + azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) + azureKeyVaultKms := expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms + } + if d.HasChanges("microsoft_defender") { updateCluster = true microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) @@ -2223,6 +2265,11 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) } d.Set("workload_identity_enabled", workloadIdentity) + azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) + if err := d.Set("key_vault_kms", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_vault_kms`: %+v", err) + } + // adminProfile is only available for RBAC enabled clusters with AAD and local account is not disabled if props.AadProfile != nil && (props.DisableLocalAccounts == nil || !*props.DisableLocalAccounts) { accessProfileId := managedclusters.NewAccessProfileID(id.SubscriptionId, id.ResourceGroupName, id.ResourceName, "clusterAdmin") @@ -3375,6 +3422,26 @@ func expandKubernetesClusterAutoScalerProfile(input []interface{}) *managedclust } } +func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.AzureKeyVaultKms { + if ((input == nil) || len(input) == 0) && d.HasChanges("key_vault_kms") { + return &managedclusters.AzureKeyVaultKms{ + Enabled: utils.Bool(false), + } + } else if (input == nil) || len(input) == 0 { + return nil + } + + raw := input[0].(map[string]interface{}) + kvAccess := managedclusters.KeyVaultNetworkAccessTypes(*utils.String(raw["key_vault_network_access"].(string))) + + return &managedclusters.AzureKeyVaultKms{ + Enabled: utils.Bool(raw["enabled"].(bool)), + KeyId: utils.String(raw["key_id"].(string)), + KeyVaultNetworkAccess: &kvAccess, + KeyVaultResourceId: utils.String(raw["key_vault_resource_id"].(string)), + } +} + func expandKubernetesClusterMaintenanceConfiguration(input []interface{}) *maintenanceconfigurations.MaintenanceConfigurationProperties { if len(input) == 0 { return nil diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 527894cb4d76..22ca7f54591c 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -75,6 +75,20 @@ func TestAccKubernetesCluster_runCommand(t *testing.T) { }) } +func TestAccKubernetesCluster_keyVaultKms(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") + r := KubernetesClusterResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.azureKeyVaultKms(data, currentKubernetesVersion), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + }) +} + func TestAccKubernetesCluster_storageProfile(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_kubernetes_cluster", "test") r := KubernetesClusterResource{} @@ -513,6 +527,43 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion, tag) } +func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string) string { + return fmt.Sprintf(` +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = "acctestRG-aks-%d" + location = "%s" +} + +resource "azurerm_kubernetes_cluster" "test" { + name = "acctestaks%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = "acctestaks%d" + kubernetes_version = %q + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "UserAssigned" + identity_ids = ["/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/acctestRG-aks-%d/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-acctestaks"] + } + + key_vault_kms { + enabled = true + key_id = "https://kv-acctestaks.vault.azure.net/keys/acctestaks/dummykeyversion" + } +} + `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion, data.RandomInteger) +} + func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/website/docs/d/kubernetes_cluster.html.markdown b/website/docs/d/kubernetes_cluster.html.markdown index 5f56663cb3ab..e626af1a3f21 100644 --- a/website/docs/d/kubernetes_cluster.html.markdown +++ b/website/docs/d/kubernetes_cluster.html.markdown @@ -56,6 +56,8 @@ The following attributes are exported: * `ingress_application_gateway` - An `ingress_application_gateway` block as documented below. +* `key_vault_kms` - A `key_vault_kms` block as documented below. + * `key_vault_secrets_provider` - A `key_vault_secrets_provider` block as documented below. * `private_fqdn` - The FQDN of this Kubernetes Cluster when private link has been enabled. This name is only resolvable inside the Virtual Network where the Azure Kubernetes Service is located @@ -178,6 +180,18 @@ A `upgrade_settings` block exports the following: --- +A `key_vault_kms` block supports the following: + +* `enabled` - Is Azure Key Vault key management service enabled? + +* `key_id` - Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. + +* `key_vault_network_access` - Network access of the key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. + +* `key_vault_resource_id` - Resource ID of key vault. This in only relevant when `key_vault_network_access` is `Private`. When keyVaultNetworkAccess is `Public`, the field is empty. + +--- + A `key_vault_secrets_provider` block exports the following: * `secret_rotation_enabled` - Is secret rotation enabled? diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 5e0a0f657c32..06dcb4c9f038 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -127,6 +127,8 @@ In addition, one of either `identity` or `service_principal` blocks must be spec * `ingress_application_gateway` - (Optional) A `ingress_application_gateway` block as defined below. +* `key_vault_kms` - (Optional) A `key_vault_kms` block as defined below. For more details, please visit [Key Management Service (KMS) etcd encryption to an AKS cluster](https://learn.microsoft.com/en-us/azure/aks/use-kms-etcd-encryption). + * `key_vault_secrets_provider` - (Optional) A `key_vault_secrets_provider` block as defined below. For more details, please visit [Azure Keyvault Secrets Provider for AKS](https://docs.microsoft.com/azure/aks/csi-secrets-store-driver). * `kubelet_identity` - (Optional) A `kubelet_identity` block as defined below. Changing this forces a new resource to be created. @@ -460,6 +462,19 @@ An `identity` block supports the following: --- +A `key_vault_kms` block supports the following: + +* `enabled` - (Optional) Whether to enable Azure Key Vault key management service. The default is `false`. + +* `key_id` - (Optional) Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. When `enabled` is `false`, leave the field empty. + +* `key_vault_network_access` - (Optional) Network access of the key vault +Network access of key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. The default value is `Public`. + +* `key_vault_resource_id` - (Optional) Resource ID of key vault. When `key_vault_network_access` is `Private`, this field is required and must be a valid resource ID. When `key_vault_network_access` is `Public`, leave the field empty. + +--- + A `key_vault_secrets_provider` block supports the following: * `secret_rotation_enabled` - (Optional) Is secret rotation enabled? From 1ba429fb206f8d9499c7d4237d805a6821ccfefa Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Wed, 4 Jan 2023 23:04:47 +0100 Subject: [PATCH 2/7] chore: Adapt construction of Defender inside securityProfile --- .../containers/kubernetes_cluster_resource.go | 46 ++++++++----------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 18909777fef5..864360eedc10 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -1364,12 +1364,15 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} publicNetworkAccess = managedclusters.PublicNetworkAccessDisabled } - microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) - securityProfile := expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) - storageProfileRaw := d.Get("storage_profile").([]interface{}) storageProfile := expandStorageProfile(storageProfileRaw) + // assemble securityProfile (Defender, WorkloadIdentity, ImageCleaner, AzureKeyVaultKms) + securityProfile := &managedclusters.ManagedClusterSecurityProfile{} + + microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) + securityProfile.Defender = expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) + workloadIdentity := false if v, ok := d.GetOk("workload_identity_enabled"); ok { workloadIdentity = v.(bool) @@ -1378,17 +1381,10 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} return fmt.Errorf("`oidc_issuer_enabled` must be set to `true` to enable Azure AD Workload Identity") } - if securityProfile == nil { - securityProfile = &managedclusters.ManagedClusterSecurityProfile{} - } - securityProfile.WorkloadIdentity = &managedclusters.ManagedClusterSecurityProfileWorkloadIdentity{ Enabled: &workloadIdentity, } } - if securityProfile == nil { - securityProfile = &managedclusters.ManagedClusterSecurityProfile{} - } securityProfile.ImageCleaner = &managedclusters.ManagedClusterSecurityProfileImageCleaner{ Enabled: utils.Bool(d.Get("image_cleaner_enabled").(bool)), @@ -1880,7 +1876,7 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} updateCluster = true microsoftDefenderRaw := d.Get("microsoft_defender").([]interface{}) microsoftDefender := expandKubernetesClusterMicrosoftDefender(d, microsoftDefenderRaw) - existing.Model.Properties.SecurityProfile = microsoftDefender + existing.Model.Properties.SecurityProfile.Defender = microsoftDefender } if d.HasChanges("storage_profile") { @@ -2249,7 +2245,7 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("oidc_issuer_enabled", oidcIssuerEnabled) d.Set("oidc_issuer_url", oidcIssuerUrl) - microsoftDefender := flattenKubernetesClusterMicrosoftDefender(props.SecurityProfile) + microsoftDefender := flattenKubernetesClusterMicrosoftDefender(props.SecurityProfile.Defender) if err := d.Set("microsoft_defender", microsoftDefender); err != nil { return fmt.Errorf("setting `microsoft_defender`: %+v", err) } @@ -3596,13 +3592,11 @@ func flattenKubernetesClusterHttpProxyConfig(props *managedclusters.ManagedClust }) } -func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.ManagedClusterSecurityProfile { +func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.ManagedClusterSecurityProfileDefender { if (len(input) == 0 || input[0] == nil) && d.HasChange("microsoft_defender") { - return &managedclusters.ManagedClusterSecurityProfile{ - Defender: &managedclusters.ManagedClusterSecurityProfileDefender{ - SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ - Enabled: utils.Bool(false), - }, + return &managedclusters.ManagedClusterSecurityProfileDefender{ + SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ + Enabled: utils.Bool(false), }, } } else if len(input) == 0 || input[0] == nil { @@ -3610,23 +3604,21 @@ func expandKubernetesClusterMicrosoftDefender(d *pluginsdk.ResourceData, input [ } config := input[0].(map[string]interface{}) - return &managedclusters.ManagedClusterSecurityProfile{ - Defender: &managedclusters.ManagedClusterSecurityProfileDefender{ - SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ - Enabled: utils.Bool(true), - }, - LogAnalyticsWorkspaceResourceId: utils.String(config["log_analytics_workspace_id"].(string)), + return &managedclusters.ManagedClusterSecurityProfileDefender{ + SecurityMonitoring: &managedclusters.ManagedClusterSecurityProfileDefenderSecurityMonitoring{ + Enabled: utils.Bool(true), }, + LogAnalyticsWorkspaceResourceId: utils.String(config["log_analytics_workspace_id"].(string)), } } -func flattenKubernetesClusterMicrosoftDefender(input *managedclusters.ManagedClusterSecurityProfile) []interface{} { - if input == nil || input.Defender == nil || (input.Defender.SecurityMonitoring != nil && input.Defender.SecurityMonitoring.Enabled != nil && !*input.Defender.SecurityMonitoring.Enabled) { +func flattenKubernetesClusterMicrosoftDefender(input *managedclusters.ManagedClusterSecurityProfileDefender) []interface{} { + if input == nil || (input.SecurityMonitoring != nil && input.SecurityMonitoring.Enabled != nil && !*input.SecurityMonitoring.Enabled) { return []interface{}{} } logAnalyticsWorkspace := "" - if v := input.Defender.LogAnalyticsWorkspaceResourceId; v != nil { + if v := input.LogAnalyticsWorkspaceResourceId; v != nil { logAnalyticsWorkspace = *v } From 77a21c2a6bfb52c5d63a915746aa4d08ccd05ce6 Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Sun, 8 Jan 2023 21:32:14 +0100 Subject: [PATCH 3/7] chore: Extend error handling and test cases --- .../containers/kubernetes_cluster_resource.go | 24 +++++-- .../kubernetes_cluster_resource_test.go | 67 ++++++++++++++++--- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 864360eedc10..0de4b94765ff 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -1392,7 +1392,10 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} } azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) - securityProfile.AzureKeyVaultKms = expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + securityProfile.AzureKeyVaultKms, err = expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + if err != nil { + return err + } parameters := managedclusters.ManagedCluster{ Name: utils.String(id.ResourceName), @@ -1868,7 +1871,7 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} if d.HasChanges("key_vault_kms") { updateCluster = true azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) - azureKeyVaultKms := expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + azureKeyVaultKms, _ := expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms } @@ -3418,24 +3421,31 @@ func expandKubernetesClusterAutoScalerProfile(input []interface{}) *managedclust } } -func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input []interface{}) *managedclusters.AzureKeyVaultKms { +func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input []interface{}) (*managedclusters.AzureKeyVaultKms, error) { if ((input == nil) || len(input) == 0) && d.HasChanges("key_vault_kms") { return &managedclusters.AzureKeyVaultKms{ Enabled: utils.Bool(false), - } + }, nil } else if (input == nil) || len(input) == 0 { - return nil + return nil, nil } raw := input[0].(map[string]interface{}) - kvAccess := managedclusters.KeyVaultNetworkAccessTypes(*utils.String(raw["key_vault_network_access"].(string))) + kvAccess := managedclusters.KeyVaultNetworkAccessTypes(raw["key_vault_network_access"].(string)) - return &managedclusters.AzureKeyVaultKms{ + azureKeyVaultKms := &managedclusters.AzureKeyVaultKms{ Enabled: utils.Bool(raw["enabled"].(bool)), KeyId: utils.String(raw["key_id"].(string)), KeyVaultNetworkAccess: &kvAccess, KeyVaultResourceId: utils.String(raw["key_vault_resource_id"].(string)), } + + private := managedclusters.KeyVaultNetworkAccessTypesPrivate + if kvAccess == private && len(*azureKeyVaultKms.KeyVaultResourceId) == 0 { + return nil, fmt.Errorf("a valid `key_vault_resource_id` is required when `key_vault_network_access` is `%s`", private) + } + + return azureKeyVaultKms, nil } func expandKubernetesClusterMaintenanceConfiguration(input []interface{}) *maintenanceconfigurations.MaintenanceConfigurationProperties { diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 22ca7f54591c..3ea7aa4a6ebc 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -81,7 +81,13 @@ func TestAccKubernetesCluster_keyVaultKms(t *testing.T) { data.ResourceTest(t, r, []acceptance.TestStep{ { - Config: r.azureKeyVaultKms(data, currentKubernetesVersion), + Config: r.azureKeyVaultKms(data, currentKubernetesVersion, true), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + { + Config: r.azureKeyVaultKms(data, currentKubernetesVersion, false), Check: acceptance.ComposeTestCheckFunc( check.That(data.ResourceName).ExistsInAzure(r), ), @@ -527,23 +533,62 @@ resource "azurerm_kubernetes_cluster" "test" { `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion, tag) } -func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string) string { +func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string, enabled bool) string { return fmt.Sprintf(` provider "azurerm" { features {} } +data "azurerm_client_config" "current" {} + resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%d" - location = "%s" + name = "acctestRG-aks-%[1]d" + location = "%[2]s" +} + +resource "azurerm_key_vault" "test" { + name = substr("acctestRG-kv-%[1]d", 0, 24) + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + tenant_id = data.azurerm_client_config.current.tenant_id + enable_rbac_authorization = true + sku_name = "standard" +} + +resource "azurerm_role_assignment" "test_admin" { + scope = azurerm_key_vault.test.id + role_definition_name = "Key Vault Administrator" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_role_assignment" "test" { + scope = azurerm_key_vault.test.id + role_definition_name = "Key Vault Crypto User" + principal_id = azurerm_user_assigned_identity.test.principal_id +} + +resource "azurerm_key_vault_key" "test" { + name = "etcd-encryption" + key_vault_id = azurerm_key_vault.test.id + key_type = "RSA" + key_size = 2048 + key_opts = ["decrypt", "encrypt", "sign", "unwrapKey", "verify", "wrapKey"] + + depends_on = [azurerm_role_assignment.test_admin] +} + +resource "azurerm_user_assigned_identity" "test" { + name = "acctest%[2]s" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location } resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%d" + name = "acctestaks%[1]d" location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%d" - kubernetes_version = %q + dns_prefix = "acctestaks%[1]d" + kubernetes_version = %[3]q default_node_pool { name = "default" @@ -553,15 +598,15 @@ resource "azurerm_kubernetes_cluster" "test" { identity { type = "UserAssigned" - identity_ids = ["/subscriptions/11111111-1111-1111-1111-111111111111/resourceGroups/acctestRG-aks-%d/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-acctestaks"] + identity_ids = [azurerm_user_assigned_identity.test.id] } key_vault_kms { - enabled = true - key_id = "https://kv-acctestaks.vault.azure.net/keys/acctestaks/dummykeyversion" + enabled = %[4]t + key_id = azurerm_key_vault_key.test.id } } - `, data.RandomInteger, data.Locations.Primary, data.RandomInteger, data.RandomInteger, controlPlaneVersion, data.RandomInteger) + `, data.RandomInteger, data.Locations.Primary, controlPlaneVersion, enabled) } func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { From 185b8f65420e671507def8aa7eab7ed741e8085d Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Mon, 9 Jan 2023 22:29:02 +0100 Subject: [PATCH 4/7] chore: Rename key_vault_kms {} -> key_management_service {} --- .../kubernetes_cluster_data_source.go | 17 +++----- .../containers/kubernetes_cluster_resource.go | 24 +++++------- .../kubernetes_cluster_resource_test.go | 39 +++++++++++++------ .../docs/d/kubernetes_cluster.html.markdown | 8 ++-- .../docs/r/kubernetes_cluster.html.markdown | 8 ++-- 5 files changed, 49 insertions(+), 47 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 30e0678a307b..56111fdca316 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -380,16 +380,12 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { "identity": commonschema.SystemOrUserAssignedIdentityComputed(), - "key_vault_kms": { + "key_management_service": { Type: pluginsdk.TypeList, Computed: true, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ - "enabled": { - Type: pluginsdk.TypeBool, - Computed: true, - }, - "key_id": { + "key_vault_key_id": { Type: pluginsdk.TypeString, Computed: true, }, @@ -744,8 +740,8 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} } azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) - if err := d.Set("key_vault_kms", azureKeyVaultKms); err != nil { - return fmt.Errorf("setting `key_vault_kms`: %+v", err) + if err := d.Set("key_management_service", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_management_service`: %+v", err) } kubeletIdentity, err := flattenKubernetesClusterDataSourceIdentityProfile(props.IdentityProfile) @@ -860,10 +856,9 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} func flattenKubernetesClusterDataSourceKeyVaultKms(input *managedclusters.AzureKeyVaultKms) []interface{} { azureKeyVaultKms := make([]interface{}, 0) - if input != nil { + if input != nil && *input.Enabled { azureKeyVaultKms = append(azureKeyVaultKms, map[string]interface{}{ - "enabled": input.Enabled, - "key_id": input.KeyId, + "key_vault_key_id": input.KeyId, "key_vault_network_access": input.KeyVaultNetworkAccess, "key_vault_resource_id": input.KeyVaultResourceId, }) diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 0de4b94765ff..0f4a4d70d9db 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -710,18 +710,14 @@ func resourceKubernetesCluster() *pluginsdk.Resource { }, }, - "key_vault_kms": { + "key_management_service": { Type: pluginsdk.TypeList, Optional: true, ForceNew: false, MaxItems: 1, Elem: &pluginsdk.Resource{ Schema: map[string]*pluginsdk.Schema{ - "enabled": { - Type: pluginsdk.TypeBool, - Required: true, - }, - "key_id": { + "key_vault_key_id": { Type: pluginsdk.TypeString, Required: true, ValidateFunc: keyVaultValidate.NestedItemId, @@ -1391,7 +1387,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} IntervalHours: utils.Int64(int64(d.Get("image_cleaner_interval_hours").(int))), } - azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) + azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) securityProfile.AzureKeyVaultKms, err = expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) if err != nil { return err @@ -1868,9 +1864,9 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} existing.Model.Properties.OidcIssuerProfile = oidcIssuerProfile } - if d.HasChanges("key_vault_kms") { + if d.HasChanges("key_management_service") { updateCluster = true - azureKeyVaultKmsRaw := d.Get("key_vault_kms").([]interface{}) + azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) azureKeyVaultKms, _ := expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms } @@ -2265,8 +2261,8 @@ func resourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{}) d.Set("workload_identity_enabled", workloadIdentity) azureKeyVaultKms := flattenKubernetesClusterDataSourceKeyVaultKms(props.SecurityProfile.AzureKeyVaultKms) - if err := d.Set("key_vault_kms", azureKeyVaultKms); err != nil { - return fmt.Errorf("setting `key_vault_kms`: %+v", err) + if err := d.Set("key_management_service", azureKeyVaultKms); err != nil { + return fmt.Errorf("setting `key_management_service`: %+v", err) } // adminProfile is only available for RBAC enabled clusters with AAD and local account is not disabled @@ -3422,7 +3418,7 @@ func expandKubernetesClusterAutoScalerProfile(input []interface{}) *managedclust } func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input []interface{}) (*managedclusters.AzureKeyVaultKms, error) { - if ((input == nil) || len(input) == 0) && d.HasChanges("key_vault_kms") { + if ((input == nil) || len(input) == 0) && d.HasChanges("key_management_service") { return &managedclusters.AzureKeyVaultKms{ Enabled: utils.Bool(false), }, nil @@ -3434,8 +3430,8 @@ func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input [] kvAccess := managedclusters.KeyVaultNetworkAccessTypes(raw["key_vault_network_access"].(string)) azureKeyVaultKms := &managedclusters.AzureKeyVaultKms{ - Enabled: utils.Bool(raw["enabled"].(bool)), - KeyId: utils.String(raw["key_id"].(string)), + Enabled: utils.Bool(true), + KeyId: utils.String(raw["key_vault_key_id"].(string)), KeyVaultNetworkAccess: &kvAccess, KeyVaultResourceId: utils.String(raw["key_vault_resource_id"].(string)), } diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 3ea7aa4a6ebc..6d6b46297128 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -1,10 +1,12 @@ package containers_test import ( + "bytes" "context" "fmt" "net/http" "testing" + "text/template" "github.com/hashicorp/go-azure-sdk/resource-manager/containerservice/2022-09-02-preview/agentpools" "github.com/hashicorp/go-azure-sdk/resource-manager/containerservice/2022-09-02-preview/managedclusters" @@ -534,7 +536,7 @@ resource "azurerm_kubernetes_cluster" "test" { } func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string, enabled bool) string { - return fmt.Sprintf(` + const hcl = ` provider "azurerm" { features {} } @@ -542,12 +544,12 @@ provider "azurerm" { data "azurerm_client_config" "current" {} resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-%[1]d" - location = "%[2]s" + name = "acctestRG-aks-{{.randomInteger}}" + location = "{{.location}}" } resource "azurerm_key_vault" "test" { - name = substr("acctestRG-kv-%[1]d", 0, 24) + name = substr("acctest{{.randomInteger}}", 0, 24) location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name tenant_id = data.azurerm_client_config.current.tenant_id @@ -578,17 +580,18 @@ resource "azurerm_key_vault_key" "test" { } resource "azurerm_user_assigned_identity" "test" { - name = "acctest%[2]s" + name = "acctest{{.randomInteger}}" resource_group_name = azurerm_resource_group.test.name location = azurerm_resource_group.test.location } resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks%[1]d" + name = "acctestaks{{.randomInteger}}" location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name - dns_prefix = "acctestaks%[1]d" - kubernetes_version = %[3]q + node_resource_group = "${azurerm_resource_group.test.name}-infra" + dns_prefix = "acctestaks{{.randomInteger}}" + kubernetes_version = "{{.controlPlaneVersion}}" default_node_pool { name = "default" @@ -601,12 +604,24 @@ resource "azurerm_kubernetes_cluster" "test" { identity_ids = [azurerm_user_assigned_identity.test.id] } - key_vault_kms { - enabled = %[4]t - key_id = azurerm_key_vault_key.test.id + {{ if .enabled }} + key_management_service { + key_vault_key_id = azurerm_key_vault_key.test.id } + {{ end }} } - `, data.RandomInteger, data.Locations.Primary, controlPlaneVersion, enabled) +` + tpl, _ := template.New("hcl").Parse(hcl) + + var tplBuffer bytes.Buffer + tpl.Execute(&tplBuffer, map[string]interface{}{ + "enabled": enabled, + "randomInteger": data.RandomInteger, + "location": data.Locations.Primary, + "controlPlaneVersion": controlPlaneVersion, + }) + + return tplBuffer.String() } func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { diff --git a/website/docs/d/kubernetes_cluster.html.markdown b/website/docs/d/kubernetes_cluster.html.markdown index e626af1a3f21..965784082fad 100644 --- a/website/docs/d/kubernetes_cluster.html.markdown +++ b/website/docs/d/kubernetes_cluster.html.markdown @@ -56,7 +56,7 @@ The following attributes are exported: * `ingress_application_gateway` - An `ingress_application_gateway` block as documented below. -* `key_vault_kms` - A `key_vault_kms` block as documented below. +* `key_management_service` - A `key_management_service` block as documented below. * `key_vault_secrets_provider` - A `key_vault_secrets_provider` block as documented below. @@ -180,11 +180,9 @@ A `upgrade_settings` block exports the following: --- -A `key_vault_kms` block supports the following: +A `key_management_service` block supports the following: -* `enabled` - Is Azure Key Vault key management service enabled? - -* `key_id` - Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. +* `key_vault_key_id` - Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. * `key_vault_network_access` - Network access of the key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index 06dcb4c9f038..e590b8611661 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -127,7 +127,7 @@ In addition, one of either `identity` or `service_principal` blocks must be spec * `ingress_application_gateway` - (Optional) A `ingress_application_gateway` block as defined below. -* `key_vault_kms` - (Optional) A `key_vault_kms` block as defined below. For more details, please visit [Key Management Service (KMS) etcd encryption to an AKS cluster](https://learn.microsoft.com/en-us/azure/aks/use-kms-etcd-encryption). +* `key_management_service` - (Optional) A `key_management_service` block as defined below. For more details, please visit [Key Management Service (KMS) etcd encryption to an AKS cluster](https://learn.microsoft.com/en-us/azure/aks/use-kms-etcd-encryption). * `key_vault_secrets_provider` - (Optional) A `key_vault_secrets_provider` block as defined below. For more details, please visit [Azure Keyvault Secrets Provider for AKS](https://docs.microsoft.com/azure/aks/csi-secrets-store-driver). @@ -462,11 +462,9 @@ An `identity` block supports the following: --- -A `key_vault_kms` block supports the following: +A `key_management_service` block supports the following: -* `enabled` - (Optional) Whether to enable Azure Key Vault key management service. The default is `false`. - -* `key_id` - (Optional) Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. When `enabled` is `false`, leave the field empty. +* `key_vault_key_id` - (Optional) Identifier of Azure Key Vault key. See [key identifier format](https://learn.microsoft.com/en-us/azure/key-vault/general/about-keys-secrets-certificates#vault-name-and-object-name) for more details. When Azure Key Vault key management service is enabled, this field is required and must be a valid key identifier. When `enabled` is `false`, leave the field empty. * `key_vault_network_access` - (Optional) Network access of the key vault Network access of key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. The default value is `Public`. From f9a7977e6ae020a401db8a1276718b363d653ecf Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Tue, 10 Jan 2023 17:33:39 +0100 Subject: [PATCH 5/7] chore: Drop "bytes" and "text/template" dependency and use common code style --- .../kubernetes_cluster_resource_test.go | 46 ++++++++----------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_resource_test.go b/internal/services/containers/kubernetes_cluster_resource_test.go index 6d6b46297128..ce48eb8bdac2 100644 --- a/internal/services/containers/kubernetes_cluster_resource_test.go +++ b/internal/services/containers/kubernetes_cluster_resource_test.go @@ -1,12 +1,10 @@ package containers_test import ( - "bytes" "context" "fmt" "net/http" "testing" - "text/template" "github.com/hashicorp/go-azure-sdk/resource-manager/containerservice/2022-09-02-preview/agentpools" "github.com/hashicorp/go-azure-sdk/resource-manager/containerservice/2022-09-02-preview/managedclusters" @@ -536,7 +534,15 @@ resource "azurerm_kubernetes_cluster" "test" { } func (KubernetesClusterResource) azureKeyVaultKms(data acceptance.TestData, controlPlaneVersion string, enabled bool) string { - const hcl = ` + kmsBlock := "" + if enabled { + kmsBlock = ` + key_management_service { + key_vault_key_id = azurerm_key_vault_key.test.id + }` + } + + return fmt.Sprintf(` provider "azurerm" { features {} } @@ -544,12 +550,12 @@ provider "azurerm" { data "azurerm_client_config" "current" {} resource "azurerm_resource_group" "test" { - name = "acctestRG-aks-{{.randomInteger}}" - location = "{{.location}}" + name = "acctestRG-aks-%[1]d" + location = "%[2]s" } resource "azurerm_key_vault" "test" { - name = substr("acctest{{.randomInteger}}", 0, 24) + name = substr("acctest%[1]d", 0, 24) location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name tenant_id = data.azurerm_client_config.current.tenant_id @@ -580,18 +586,18 @@ resource "azurerm_key_vault_key" "test" { } resource "azurerm_user_assigned_identity" "test" { - name = "acctest{{.randomInteger}}" + name = "acctest%[1]d" resource_group_name = azurerm_resource_group.test.name location = azurerm_resource_group.test.location } resource "azurerm_kubernetes_cluster" "test" { - name = "acctestaks{{.randomInteger}}" + name = "acctestaks%[1]d" location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name node_resource_group = "${azurerm_resource_group.test.name}-infra" - dns_prefix = "acctestaks{{.randomInteger}}" - kubernetes_version = "{{.controlPlaneVersion}}" + dns_prefix = "acctestaks%[1]d" + kubernetes_version = %[3]q default_node_pool { name = "default" @@ -603,25 +609,9 @@ resource "azurerm_kubernetes_cluster" "test" { type = "UserAssigned" identity_ids = [azurerm_user_assigned_identity.test.id] } - - {{ if .enabled }} - key_management_service { - key_vault_key_id = azurerm_key_vault_key.test.id - } - {{ end }} + %[4]s } -` - tpl, _ := template.New("hcl").Parse(hcl) - - var tplBuffer bytes.Buffer - tpl.Execute(&tplBuffer, map[string]interface{}{ - "enabled": enabled, - "randomInteger": data.RandomInteger, - "location": data.Locations.Primary, - "controlPlaneVersion": controlPlaneVersion, - }) - - return tplBuffer.String() +`, data.RandomInteger, data.Locations.Primary, controlPlaneVersion, kmsBlock) } func (KubernetesClusterResource) storageProfile(data acceptance.TestData, controlPlaneVersion string) string { From ffca49ea78a7598c077df3b7cde77f33129c4564 Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Fri, 13 Jan 2023 18:20:26 +0100 Subject: [PATCH 6/7] chore: Automatically set key vault resource ID --- .../kubernetes_cluster_data_source.go | 5 --- .../containers/kubernetes_cluster_resource.go | 34 ++++++++++++------- .../docs/d/kubernetes_cluster.html.markdown | 2 -- .../docs/r/kubernetes_cluster.html.markdown | 2 -- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 56111fdca316..46eab07e9cec 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -393,10 +393,6 @@ func dataSourceKubernetesCluster() *pluginsdk.Resource { Type: pluginsdk.TypeString, Computed: true, }, - "key_vault_resource_id": { - Type: pluginsdk.TypeString, - Computed: true, - }, }, }, }, @@ -860,7 +856,6 @@ func flattenKubernetesClusterDataSourceKeyVaultKms(input *managedclusters.AzureK azureKeyVaultKms = append(azureKeyVaultKms, map[string]interface{}{ "key_vault_key_id": input.KeyId, "key_vault_network_access": input.KeyVaultNetworkAccess, - "key_vault_resource_id": input.KeyVaultResourceId, }) } diff --git a/internal/services/containers/kubernetes_cluster_resource.go b/internal/services/containers/kubernetes_cluster_resource.go index 0f4a4d70d9db..0663f26b9d1f 100644 --- a/internal/services/containers/kubernetes_cluster_resource.go +++ b/internal/services/containers/kubernetes_cluster_resource.go @@ -31,8 +31,11 @@ import ( "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/kubernetes" "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/migration" containerValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/containers/validate" + keyVaultClient "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/client" + keyVaultParse "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/parse" keyVaultValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/keyvault/validate" networkValidate "github.com/hashicorp/terraform-provider-azurerm/internal/services/network/validate" + resourcesClient "github.com/hashicorp/terraform-provider-azurerm/internal/services/resource/client" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/suppress" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/validation" @@ -728,11 +731,6 @@ func resourceKubernetesCluster() *pluginsdk.Resource { Optional: true, ValidateFunc: validation.StringInSlice(managedclusters.PossibleValuesForKeyVaultNetworkAccessTypes(), false), }, - "key_vault_resource_id": { - Type: pluginsdk.TypeString, - Optional: true, - ValidateFunc: keyVaultValidate.VaultID, - }, }, }, }, @@ -1253,6 +1251,8 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} subscriptionId := meta.(*clients.Client).Account.SubscriptionId tenantId := meta.(*clients.Client).Account.TenantId client := meta.(*clients.Client).Containers.KubernetesClustersClient + keyVaultsClient := meta.(*clients.Client).KeyVault + resourcesClient := meta.(*clients.Client).Resource env := meta.(*clients.Client).Containers.Environment ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -1388,7 +1388,7 @@ func resourceKubernetesClusterCreate(d *pluginsdk.ResourceData, meta interface{} } azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) - securityProfile.AzureKeyVaultKms, err = expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + securityProfile.AzureKeyVaultKms, err = expandKubernetesClusterAzureKeyVaultKms(ctx, keyVaultsClient, resourcesClient, d, azureKeyVaultKmsRaw) if err != nil { return err } @@ -1518,6 +1518,8 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} containersClient := meta.(*clients.Client).Containers nodePoolsClient := containersClient.AgentPoolsClient clusterClient := containersClient.KubernetesClustersClient + keyVaultsClient := meta.(*clients.Client).KeyVault + resourcesClient := meta.(*clients.Client).Resource env := containersClient.Environment ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -1867,7 +1869,7 @@ func resourceKubernetesClusterUpdate(d *pluginsdk.ResourceData, meta interface{} if d.HasChanges("key_management_service") { updateCluster = true azureKeyVaultKmsRaw := d.Get("key_management_service").([]interface{}) - azureKeyVaultKms, _ := expandKubernetesClusterAzureKeyVaultKms(d, azureKeyVaultKmsRaw) + azureKeyVaultKms, _ := expandKubernetesClusterAzureKeyVaultKms(ctx, keyVaultsClient, resourcesClient, d, azureKeyVaultKmsRaw) existing.Model.Properties.SecurityProfile.AzureKeyVaultKms = azureKeyVaultKms } @@ -3417,7 +3419,7 @@ func expandKubernetesClusterAutoScalerProfile(input []interface{}) *managedclust } } -func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input []interface{}) (*managedclusters.AzureKeyVaultKms, error) { +func expandKubernetesClusterAzureKeyVaultKms(ctx context.Context, keyVaultsClient *keyVaultClient.Client, resourcesClient *resourcesClient.Client, d *pluginsdk.ResourceData, input []interface{}) (*managedclusters.AzureKeyVaultKms, error) { if ((input == nil) || len(input) == 0) && d.HasChanges("key_management_service") { return &managedclusters.AzureKeyVaultKms{ Enabled: utils.Bool(false), @@ -3433,12 +3435,20 @@ func expandKubernetesClusterAzureKeyVaultKms(d *pluginsdk.ResourceData, input [] Enabled: utils.Bool(true), KeyId: utils.String(raw["key_vault_key_id"].(string)), KeyVaultNetworkAccess: &kvAccess, - KeyVaultResourceId: utils.String(raw["key_vault_resource_id"].(string)), } - private := managedclusters.KeyVaultNetworkAccessTypesPrivate - if kvAccess == private && len(*azureKeyVaultKms.KeyVaultResourceId) == 0 { - return nil, fmt.Errorf("a valid `key_vault_resource_id` is required when `key_vault_network_access` is `%s`", private) + // Set Key vault Resource ID in case public access is disabled + if kvAccess == managedclusters.KeyVaultNetworkAccessTypesPrivate { + keyVaultKeyId, err := keyVaultParse.ParseNestedItemID(*azureKeyVaultKms.KeyId) + if err != nil { + return nil, err + } + keyVaultID, err := keyVaultsClient.KeyVaultIDFromBaseUrl(ctx, resourcesClient, keyVaultKeyId.KeyVaultBaseUrl) + if err != nil { + return nil, fmt.Errorf("retrieving the Resource ID the Key Vault at URL %q: %s", keyVaultKeyId.KeyVaultBaseUrl, err) + } + + azureKeyVaultKms.KeyVaultResourceId = keyVaultID } return azureKeyVaultKms, nil diff --git a/website/docs/d/kubernetes_cluster.html.markdown b/website/docs/d/kubernetes_cluster.html.markdown index 965784082fad..97031973b07e 100644 --- a/website/docs/d/kubernetes_cluster.html.markdown +++ b/website/docs/d/kubernetes_cluster.html.markdown @@ -186,8 +186,6 @@ A `key_management_service` block supports the following: * `key_vault_network_access` - Network access of the key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. -* `key_vault_resource_id` - Resource ID of key vault. This in only relevant when `key_vault_network_access` is `Private`. When keyVaultNetworkAccess is `Public`, the field is empty. - --- A `key_vault_secrets_provider` block exports the following: diff --git a/website/docs/r/kubernetes_cluster.html.markdown b/website/docs/r/kubernetes_cluster.html.markdown index e590b8611661..e65077a94b5b 100644 --- a/website/docs/r/kubernetes_cluster.html.markdown +++ b/website/docs/r/kubernetes_cluster.html.markdown @@ -469,8 +469,6 @@ A `key_management_service` block supports the following: * `key_vault_network_access` - (Optional) Network access of the key vault Network access of key vault. The possible values are `Public` and `Private`. `Public` means the key vault allows public access from all networks. `Private` means the key vault disables public access and enables private link. The default value is `Public`. -* `key_vault_resource_id` - (Optional) Resource ID of key vault. When `key_vault_network_access` is `Private`, this field is required and must be a valid resource ID. When `key_vault_network_access` is `Public`, leave the field empty. - --- A `key_vault_secrets_provider` block supports the following: From 4636aa43b55e3508f708f1727ce8e27bed46fcc8 Mon Sep 17 00:00:00 2001 From: Marco Kilchhofer Date: Mon, 16 Jan 2023 22:14:16 +0100 Subject: [PATCH 7/7] chore: Add nil check on input attributes Apply suggestions from code review Co-authored-by: stephybun --- .../containers/kubernetes_cluster_data_source.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/services/containers/kubernetes_cluster_data_source.go b/internal/services/containers/kubernetes_cluster_data_source.go index 46eab07e9cec..742fb432f92c 100644 --- a/internal/services/containers/kubernetes_cluster_data_source.go +++ b/internal/services/containers/kubernetes_cluster_data_source.go @@ -852,10 +852,20 @@ func dataSourceKubernetesClusterRead(d *pluginsdk.ResourceData, meta interface{} func flattenKubernetesClusterDataSourceKeyVaultKms(input *managedclusters.AzureKeyVaultKms) []interface{} { azureKeyVaultKms := make([]interface{}, 0) - if input != nil && *input.Enabled { + if input != nil && input.Enabled != nil && *input.Enabled { + keyId := "" + if v := input.KeyId; v != nil { + keyId = *v + } + + networkAccess := "" + if v := input.KeyVaultNetworkAccess; v != nil { + networkAccess = string(*v) + } + azureKeyVaultKms = append(azureKeyVaultKms, map[string]interface{}{ - "key_vault_key_id": input.KeyId, - "key_vault_network_access": input.KeyVaultNetworkAccess, + "key_vault_key_id": keyId, + "key_vault_network_access": networkAccess, }) }