From e101fa3a7832fb008ddcaa6e8ec6592afca77e3d Mon Sep 17 00:00:00 2001 From: Jiawei Tao Date: Tue, 16 Apr 2024 14:06:48 +0800 Subject: [PATCH 1/2] HDInsight: add support for private_link_configuration --- .../hdinsight_hadoop_cluster_resource.go | 20 +- .../hdinsight_hadoop_cluster_resource_test.go | 17 +- .../hdinsight_hbase_cluster_resource.go | 20 +- .../hdinsight_hbase_cluster_resource_test.go | 11 + ...ight_interactive_query_cluster_resource.go | 20 +- ...interactive_query_cluster_resource_test.go | 11 + .../hdinsight_kafka_cluster_resource.go | 19 +- .../hdinsight_kafka_cluster_resource_test.go | 361 ++++++++++++++++++ .../hdinsight_spark_cluster_resource.go | 14 +- .../hdinsight_spark_cluster_resource_test.go | 11 + internal/services/hdinsight/schema.go | 164 ++++++++ .../r/hdinsight_hadoop_cluster.html.markdown | 26 ++ .../r/hdinsight_hbase_cluster.html.markdown | 26 ++ ...ht_interactive_query_cluster.html.markdown | 26 ++ .../r/hdinsight_kafka_cluster.html.markdown | 26 ++ .../r/hdinsight_spark_cluster.html.markdown | 26 ++ 16 files changed, 775 insertions(+), 23 deletions(-) diff --git a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go index 99c958271cb7..067484d622a4 100644 --- a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go @@ -120,6 +120,8 @@ func resourceHDInsightHadoopCluster() *pluginsdk.Resource { "storage_account_gen2": SchemaHDInsightsGen2StorageAccounts(), + "private_link_configuration": SchemaHDInsightPrivateLinkConfigurations(), + "roles": { Type: pluginsdk.TypeList, Required: true, @@ -233,6 +235,9 @@ func resourceHDInsightHadoopClusterCreate(d *pluginsdk.ResourceData, meta interf networkPropertiesRaw := d.Get("network").([]interface{}) networkProperties := ExpandHDInsightsNetwork(networkPropertiesRaw) + privateLinkConfigurationsRaw := d.Get("private_link_configuration").([]interface{}) + privateLinkConfigurations := ExpandHDInsightPrivateLinkConfigurations(privateLinkConfigurationsRaw) + computeIsolationProperties := ExpandHDInsightComputeIsolationProperties(d.Get("compute_isolation").([]interface{})) storageAccountsRaw := d.Get("storage_account").([]interface{}) @@ -268,11 +273,12 @@ func resourceHDInsightHadoopClusterCreate(d *pluginsdk.ResourceData, meta interf payload := clusters.ClusterCreateParametersExtended{ Location: utils.String(location), Properties: &clusters.ClusterCreateProperties{ - Tier: pointer.To(tier), - OsType: pointer.To(clusters.OSTypeLinux), - ClusterVersion: utils.String(clusterVersion), - MinSupportedTlsVersion: utils.String(tls), - NetworkProperties: networkProperties, + Tier: pointer.To(tier), + OsType: pointer.To(clusters.OSTypeLinux), + ClusterVersion: utils.String(clusterVersion), + MinSupportedTlsVersion: utils.String(tls), + NetworkProperties: networkProperties, + PrivateLinkConfigurations: privateLinkConfigurations, ClusterDefinition: &clusters.ClusterDefinition{ Kind: pointer.To(clusters.ClusterKindHadoop), ComponentVersion: pointer.To(componentVersions), @@ -440,6 +446,10 @@ func resourceHDInsightHadoopClusterRead(d *pluginsdk.ResourceData, meta interfac return fmt.Errorf("flattening `network`: %+v", err) } + if err := d.Set("private_link_configuration", flattenHDInsightPrivateLinkConfigurations(props.PrivateLinkConfigurations)); err != nil { + return fmt.Errorf("flattening `private_link_configuration`: %+v", err) + } + hadoopRoles := hdInsightRoleDefinition{ HeadNodeDef: hdInsightHadoopClusterHeadNodeDefinition, WorkerNodeDef: hdInsightHadoopClusterWorkerNodeDefinition, diff --git a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go index 660d616f703f..42661e5c61cd 100644 --- a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go @@ -1507,6 +1507,17 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" { is_default = true } + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + roles { head_node { vm_size = "Standard_D3_V2" @@ -1784,7 +1795,11 @@ resource "azurerm_storage_container" "test" { func (HDInsightHadoopClusterResource) gen2template(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { - features {} + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } } resource "azurerm_resource_group" "test" { diff --git a/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go b/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go index f917a2a24610..e104e120a016 100644 --- a/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go @@ -111,6 +111,8 @@ func resourceHDInsightHBaseCluster() *pluginsdk.Resource { "storage_account_gen2": SchemaHDInsightsGen2StorageAccounts(), + "private_link_configuration": SchemaHDInsightPrivateLinkConfigurations(), + "roles": { Type: pluginsdk.TypeList, Required: true, @@ -181,6 +183,9 @@ func resourceHDInsightHBaseClusterCreate(d *pluginsdk.ResourceData, meta interfa networkPropertiesRaw := d.Get("network").([]interface{}) networkProperties := ExpandHDInsightsNetwork(networkPropertiesRaw) + privateLinkConfigurationsRaw := d.Get("private_link_configuration").([]interface{}) + privateLinkConfigurations := ExpandHDInsightPrivateLinkConfigurations(privateLinkConfigurationsRaw) + hbaseRoles := hdInsightRoleDefinition{ HeadNodeDef: hdInsightHBaseClusterHeadNodeDefinition, WorkerNodeDef: hdInsightHBaseClusterWorkerNodeDefinition, @@ -209,11 +214,12 @@ func resourceHDInsightHBaseClusterCreate(d *pluginsdk.ResourceData, meta interfa params := clusters.ClusterCreateParametersExtended{ Location: utils.String(location), Properties: &clusters.ClusterCreateProperties{ - Tier: pointer.To(tier), - OsType: pointer.To(clusters.OSTypeLinux), - ClusterVersion: utils.String(clusterVersion), - MinSupportedTlsVersion: utils.String(tls), - NetworkProperties: networkProperties, + Tier: pointer.To(tier), + OsType: pointer.To(clusters.OSTypeLinux), + ClusterVersion: utils.String(clusterVersion), + MinSupportedTlsVersion: utils.String(tls), + NetworkProperties: networkProperties, + PrivateLinkConfigurations: privateLinkConfigurations, ClusterDefinition: &clusters.ClusterDefinition{ Kind: pointer.To(clusters.ClusterKindHBase), ComponentVersion: pointer.To(componentVersions), @@ -352,6 +358,10 @@ func resourceHDInsightHBaseClusterRead(d *pluginsdk.ResourceData, meta interface return fmt.Errorf("flattening `network`: %+v", err) } + if err := d.Set("private_link_configuration", flattenHDInsightPrivateLinkConfigurations(props.PrivateLinkConfigurations)); err != nil { + return fmt.Errorf("flattening `private_link_configuration`: %+v", err) + } + diskEncryptionProps, err := flattenHDInsightsDiskEncryptionProperties(props.DiskEncryptionProperties) if err != nil { return err diff --git a/internal/services/hdinsight/hdinsight_hbase_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_hbase_cluster_resource_test.go index cfeb0a08d6b4..d029d35defd3 100644 --- a/internal/services/hdinsight/hdinsight_hbase_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_hbase_cluster_resource_test.go @@ -834,6 +834,17 @@ resource "azurerm_hdinsight_hbase_cluster" "test" { is_default = true } + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + roles { head_node { vm_size = "Standard_D3_V2" diff --git a/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go b/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go index a3f651e0e56d..cfdb3040867c 100644 --- a/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go @@ -120,6 +120,8 @@ func resourceHDInsightInteractiveQueryCluster() *pluginsdk.Resource { "storage_account_gen2": SchemaHDInsightsGen2StorageAccounts(), + "private_link_configuration": SchemaHDInsightPrivateLinkConfigurations(), + "tags": commonschema.Tags(), "https_endpoint": { @@ -205,6 +207,9 @@ func resourceHDInsightInteractiveQueryClusterCreate(d *pluginsdk.ResourceData, m networkPropertiesRaw := d.Get("network").([]interface{}) networkProperties := ExpandHDInsightsNetwork(networkPropertiesRaw) + privateLinkConfigurationsRaw := d.Get("private_link_configuration").([]interface{}) + privateLinkConfigurations := ExpandHDInsightPrivateLinkConfigurations(privateLinkConfigurationsRaw) + storageAccountsRaw := d.Get("storage_account").([]interface{}) storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{}) storageAccounts, expandedIdentity, err := ExpandHDInsightsStorageAccounts(storageAccountsRaw, storageAccountsGen2Raw) @@ -242,11 +247,12 @@ func resourceHDInsightInteractiveQueryClusterCreate(d *pluginsdk.ResourceData, m params := clusters.ClusterCreateParametersExtended{ Location: utils.String(location), Properties: &clusters.ClusterCreateProperties{ - Tier: pointer.To(tier), - OsType: pointer.To(clusters.OSTypeLinux), - ClusterVersion: utils.String(clusterVersion), - MinSupportedTlsVersion: utils.String(tls), - NetworkProperties: networkProperties, + Tier: pointer.To(tier), + OsType: pointer.To(clusters.OSTypeLinux), + ClusterVersion: utils.String(clusterVersion), + MinSupportedTlsVersion: utils.String(tls), + NetworkProperties: networkProperties, + PrivateLinkConfigurations: privateLinkConfigurations, EncryptionInTransitProperties: &clusters.EncryptionInTransitProperties{ IsEncryptionInTransitEnabled: &encryptionInTransit, }, @@ -400,6 +406,10 @@ func resourceHDInsightInteractiveQueryClusterRead(d *pluginsdk.ResourceData, met return fmt.Errorf("flattening `network`: %+v", err) } + if err := d.Set("private_link_configuration", flattenHDInsightPrivateLinkConfigurations(props.PrivateLinkConfigurations)); err != nil { + return fmt.Errorf("flattening `private_link_configuration`: %+v", err) + } + interactiveQueryRoles := hdInsightRoleDefinition{ HeadNodeDef: hdInsightInteractiveQueryClusterHeadNodeDefinition, WorkerNodeDef: hdInsightInteractiveQueryClusterWorkerNodeDefinition, diff --git a/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource_test.go index a4ab34ae9e45..19854efecb87 100644 --- a/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource_test.go @@ -881,6 +881,17 @@ resource "azurerm_hdinsight_interactive_query_cluster" "test" { is_default = true } + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + roles { head_node { vm_size = "Standard_D13_V2" diff --git a/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go b/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go index 12b623c46a8d..ee95e47c0410 100644 --- a/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go @@ -116,6 +116,8 @@ func resourceHDInsightKafkaCluster() *pluginsdk.Resource { "storage_account_gen2": SchemaHDInsightsGen2StorageAccounts(), + "private_link_configuration": SchemaHDInsightPrivateLinkConfigurations(), + "compute_isolation": SchemaHDInsightsComputeIsolation(), "encryption_in_transit_enabled": { @@ -270,6 +272,9 @@ func resourceHDInsightKafkaClusterCreate(d *pluginsdk.ResourceData, meta interfa networkPropertiesRaw := d.Get("network").([]interface{}) networkProperties := ExpandHDInsightsNetwork(networkPropertiesRaw) + privateLinkConfigurationsRaw := d.Get("private_link_configuration").([]interface{}) + privateLinkConfigurations := ExpandHDInsightPrivateLinkConfigurations(privateLinkConfigurationsRaw) + kafkaRoles := hdInsightRoleDefinition{ HeadNodeDef: hdInsightKafkaClusterHeadNodeDefinition, WorkerNodeDef: hdInsightKafkaClusterWorkerNodeDefinition, @@ -301,11 +306,12 @@ func resourceHDInsightKafkaClusterCreate(d *pluginsdk.ResourceData, meta interfa payload := clusters.ClusterCreateParametersExtended{ Location: utils.String(location), Properties: &clusters.ClusterCreateProperties{ - Tier: pointer.To(tier), - OsType: pointer.To(clusters.OSTypeLinux), - ClusterVersion: utils.String(clusterVersion), - MinSupportedTlsVersion: utils.String(tls), - NetworkProperties: networkProperties, + Tier: pointer.To(tier), + OsType: pointer.To(clusters.OSTypeLinux), + ClusterVersion: utils.String(clusterVersion), + MinSupportedTlsVersion: utils.String(tls), + NetworkProperties: networkProperties, + PrivateLinkConfigurations: privateLinkConfigurations, ClusterDefinition: &clusters.ClusterDefinition{ Kind: pointer.To(clusters.ClusterKindKafka), ComponentVersion: pointer.To(componentVersions), @@ -472,6 +478,9 @@ func resourceHDInsightKafkaClusterRead(d *pluginsdk.ResourceData, meta interface if err := d.Set("network", flattenHDInsightsNetwork(props.NetworkProperties)); err != nil { return fmt.Errorf("flatten `network`: %+v", err) } + if err := d.Set("private_link_configuration", flattenHDInsightPrivateLinkConfigurations(props.PrivateLinkConfigurations)); err != nil { + return fmt.Errorf("flattening `private_link_configuration`: %+v", err) + } if err := d.Set("compute_isolation", flattenHDInsightComputeIsolationProperties(props.ComputeIsolationProperties)); err != nil { return fmt.Errorf("failed setting `compute_isolation`: %+v", err) } diff --git a/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go index a71c976cda50..f40705b5ba4f 100644 --- a/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go @@ -62,6 +62,26 @@ func TestAccHDInsightKafkaCluster_gen2storage(t *testing.T) { }) } +func TestAccHDInsightKafkaCluster_privateLink(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_hdinsight_kafka_cluster", "test") + r := HDInsightKafkaClusterResource{} + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.privateLink(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("roles.0.head_node.0.password", + "roles.0.head_node.0.vm_size", + "roles.0.worker_node.0.password", + "roles.0.worker_node.0.vm_size", + "roles.0.zookeeper_node.0.password", + "roles.0.zookeeper_node.0.vm_size", + "storage_account"), + }) +} + func TestAccHDInsightKafkaCluster_requiresImport(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_hdinsight_kafka_cluster", "test") r := HDInsightKafkaClusterResource{} @@ -710,6 +730,135 @@ resource "azurerm_hdinsight_kafka_cluster" "test" { `, r.gen2template(data), data.RandomInteger) } +func (r HDInsightKafkaClusterResource) privateLink(data acceptance.TestData) string { + return fmt.Sprintf(` +%s + +resource "azurerm_virtual_network" "test" { + name = "acctestvirtnet%d" + address_space = ["172.16.0.0/16"] + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name +} + +resource "azurerm_subnet" "test" { + name = "acctestsubnet%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["172.16.11.0/26"] + + enforce_private_link_service_network_policies = true +} + +resource "azurerm_public_ip" "test" { + name = "acctestpip%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + allocation_method = "Static" + sku = "Standard" + zones = ["1"] +} + +resource "azurerm_nat_gateway" "test" { + name = "acctestnat%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + sku_name = "Standard" + idle_timeout_in_minutes = 10 + zones = ["1"] +} + +resource "azurerm_nat_gateway_public_ip_association" "test" { + nat_gateway_id = azurerm_nat_gateway.test.id + public_ip_address_id = azurerm_public_ip.test.id +} + +resource "azurerm_subnet_nat_gateway_association" "test" { + subnet_id = azurerm_subnet.test.id + nat_gateway_id = azurerm_nat_gateway.test.id +} + +resource "azurerm_subnet_network_security_group_association" "test" { + subnet_id = azurerm_subnet.test.id + network_security_group_id = azurerm_network_security_group.test.id +} + +resource "azurerm_hdinsight_kafka_cluster" "test" { + depends_on = [azurerm_role_assignment.test, azurerm_nat_gateway.test, azurerm_subnet_network_security_group_association.test] + + name = "acctesthdi-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + cluster_version = "4.0" + tier = "Standard" + + component_version { + hadoop = "3.1" + } + + network { + connection_direction = "Outbound" + private_link_enabled = true + } + + gateway { + username = "acctestusrgw" + password = "TerrAform123!" + } + + storage_account_gen2 { + storage_resource_id = azurerm_storage_account.gen2test.id + filesystem_id = azurerm_storage_data_lake_gen2_filesystem.gen2test.id + managed_identity_resource_id = azurerm_user_assigned_identity.test.id + is_default = true + } + + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + + roles { + head_node { + vm_size = "Standard_D3_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + + worker_node { + vm_size = "Standard_D4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 3 + + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + + zookeeper_node { + vm_size = "Standard_D3_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + } +} + +%s +`, r.gen2template(data), data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger, r.nsgTemplate(data)) +} + func (r HDInsightKafkaClusterResource) requiresImport(data acceptance.TestData) string { return fmt.Sprintf(` %s @@ -1039,6 +1188,218 @@ resource "azurerm_hdinsight_kafka_cluster" "test" { `, r.template(data), data.RandomInteger, data.RandomInteger, data.RandomInteger) } +func (HDInsightKafkaClusterResource) nsgTemplate(data acceptance.TestData) string { + return fmt.Sprintf(` +resource "azurerm_network_security_group" "test" { + name = "acctestnsg-%d" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + + security_rule = [ + { + access = "Allow" + description = "Rule can be deleted but do not change source ips." + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "443" + destination_port_ranges = [] + direction = "Inbound" + name = "Rule-101" + priority = 101 + protocol = "Tcp" + source_address_prefix = "VirtualNetwork" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Allow" + description = "Rule can be deleted but do not change source ips." + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "*" + destination_port_ranges = [] + direction = "Inbound" + name = "Rule-103" + priority = 103 + protocol = "*" + source_address_prefix = "CorpNetPublic" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Allow" + description = "Rule can be deleted but do not change source ips." + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "*" + destination_port_ranges = [] + direction = "Inbound" + name = "Rule-104" + priority = 104 + protocol = "*" + source_address_prefix = "CorpNetSaw" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Deny" + description = "DO NOT DELETE" + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "" + destination_port_ranges = [ + "111", + "11211", + "123", + "13", + "17", + "19", + "1900", + "512", + "514", + "53", + "5353", + "593", + "69", + "873", + ] + direction = "Inbound" + name = "Rule-108" + priority = 108 + protocol = "*" + source_address_prefix = "Internet" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Deny" + description = "DO NOT DELETE" + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "" + destination_port_ranges = [ + "119", + "137", + "138", + "139", + "161", + "162", + "2049", + "2301", + "2381", + "3268", + "389", + "5800", + "5900", + "636", + ] + direction = "Inbound" + name = "Rule-109" + priority = 109 + protocol = "*" + source_address_prefix = "Internet" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Deny" + description = "DO NOT DELETE" + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "" + destination_port_ranges = [ + "135", + "23", + "445", + "5985", + "5986", + ] + direction = "Inbound" + name = "Rule-107" + priority = 107 + protocol = "Tcp" + source_address_prefix = "Internet" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Deny" + description = "DO NOT DELETE" + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "" + destination_port_ranges = [ + "1433", + "1434", + "16379", + "26379", + "27017", + "3306", + "4333", + "5432", + "6379", + "7000", + "7001", + "7199", + "9042", + "9160", + "9300", + ] + direction = "Inbound" + name = "Rule-105" + priority = 105 + protocol = "*" + source_address_prefix = "Internet" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + { + access = "Deny" + description = "DO NOT DELETE" + destination_address_prefix = "*" + destination_address_prefixes = [] + destination_application_security_group_ids = [] + destination_port_range = "" + destination_port_ranges = [ + "22", + "3389", + ] + direction = "Inbound" + name = "Rule-106" + priority = 106 + protocol = "Tcp" + source_address_prefix = "Internet" + source_address_prefixes = [] + source_application_security_group_ids = [] + source_port_range = "*" + source_port_ranges = [] + }, + ] +} +`, data.RandomInteger) +} + func (HDInsightKafkaClusterResource) template(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { diff --git a/internal/services/hdinsight/hdinsight_spark_cluster_resource.go b/internal/services/hdinsight/hdinsight_spark_cluster_resource.go index fbc91c37a025..1f5284ab77ec 100644 --- a/internal/services/hdinsight/hdinsight_spark_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_spark_cluster_resource.go @@ -119,6 +119,8 @@ func resourceHDInsightSparkCluster() *pluginsdk.Resource { "storage_account_gen2": SchemaHDInsightsGen2StorageAccounts(), + "private_link_configuration": SchemaHDInsightPrivateLinkConfigurations(), + "roles": { Type: pluginsdk.TypeList, Required: true, @@ -190,6 +192,9 @@ func resourceHDInsightSparkClusterCreate(d *pluginsdk.ResourceData, meta interfa networkPropertiesRaw := d.Get("network").([]interface{}) networkProperties := ExpandHDInsightsNetwork(networkPropertiesRaw) + privateLinkConfigurationsRaw := d.Get("private_link_configuration").([]interface{}) + privateLinkConfigurations := ExpandHDInsightPrivateLinkConfigurations(privateLinkConfigurationsRaw) + sparkRoles := hdInsightRoleDefinition{ HeadNodeDef: hdInsightSparkClusterHeadNodeDefinition, WorkerNodeDef: hdInsightSparkClusterWorkerNodeDefinition, @@ -226,8 +231,9 @@ func resourceHDInsightSparkClusterCreate(d *pluginsdk.ResourceData, meta interfa EncryptionInTransitProperties: &clusters.EncryptionInTransitProperties{ IsEncryptionInTransitEnabled: &encryptionInTransit, }, - MinSupportedTlsVersion: utils.String(tls), - NetworkProperties: networkProperties, + MinSupportedTlsVersion: utils.String(tls), + NetworkProperties: networkProperties, + PrivateLinkConfigurations: privateLinkConfigurations, ClusterDefinition: &clusters.ClusterDefinition{ Kind: pointer.To(clusters.ClusterKindSpark), ComponentVersion: pointer.To(componentVersions), @@ -383,6 +389,10 @@ func resourceHDInsightSparkClusterRead(d *pluginsdk.ResourceData, meta interface return fmt.Errorf("flattening `network`: %+v", err) } + if err := d.Set("private_link_configuration", flattenHDInsightPrivateLinkConfigurations(props.PrivateLinkConfigurations)); err != nil { + return fmt.Errorf("flattening `private_link_configuration`: %+v", err) + } + flattenedRoles := flattenHDInsightRoles(d, props.ComputeProfile, sparkRoles) if err := d.Set("roles", flattenedRoles); err != nil { return fmt.Errorf("flattening `roles`: %+v", err) diff --git a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go index f1d6fe10d2ca..ba5ff390e8fb 100644 --- a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go @@ -933,6 +933,17 @@ resource "azurerm_hdinsight_spark_cluster" "test" { is_default = true } + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + roles { head_node { vm_size = "Standard_D13_V2" diff --git a/internal/services/hdinsight/schema.go b/internal/services/hdinsight/schema.go index 3aeb69702bbc..36c461cbdc34 100644 --- a/internal/services/hdinsight/schema.go +++ b/internal/services/hdinsight/schema.go @@ -584,6 +584,66 @@ func ExpandHDInsightsNetwork(input []interface{}) *clusters.NetworkProperties { } } +func ExpandHDInsightPrivateLinkConfigurations(input []interface{}) *[]clusters.PrivateLinkConfiguration { + if len(input) == 0 { + return nil + } + + configs := make([]clusters.PrivateLinkConfiguration, 0) + + for _, vs := range input { + v := vs.(map[string]interface{}) + + configs = append(configs, clusters.PrivateLinkConfiguration{ + Name: v["name"].(string), + Properties: ExpandHDInsightPrivateLinkConfigurationProperties(input), + }) + } + + return pointer.To(configs) +} + +func ExpandHDInsightPrivateLinkConfigurationProperties(input []interface{}) clusters.PrivateLinkConfigurationProperties { + v := input[0].(map[string]interface{}) + + return clusters.PrivateLinkConfigurationProperties{ + GroupId: v["group_id"].(string), + IPConfigurations: ExpandHDInsightPrivateLinkConfigurationIpConfiguration(v["ip_configuration"].([]interface{})), + } +} + +func ExpandHDInsightPrivateLinkConfigurationIpConfiguration(input []interface{}) []clusters.IPConfiguration { + ipConfigs := make([]clusters.IPConfiguration, 0) + + for _, vs := range input { + v := vs.(map[string]interface{}) + + ipConfigs = append(ipConfigs, clusters.IPConfiguration{ + Name: v["name"].(string), + Properties: ExpandHDInsightPrivateLinkConfigurationIpConfigurationProperties(input), + }) + } + + return ipConfigs +} + +func ExpandHDInsightPrivateLinkConfigurationIpConfigurationProperties(input []interface{}) *clusters.IPConfigurationProperties { + v := input[0].(map[string]interface{}) + + props := clusters.IPConfigurationProperties{ + Primary: pointer.To(v["primary"].(bool)), + PrivateIPAllocationMethod: pointer.To(clusters.PrivateIPAllocationMethod(v["private_ip_allocation_method"].(string))), + } + if v["private_ip_address"] != nil && v["private_ip_address"].(string) != "" { + props.PrivateIPAddress = pointer.To(v["private_ip_address"].(string)) + } + if v["subnet_id"] != nil && v["subnet_id"].(string) != "" { + props.Subnet = pointer.To(clusters.ResourceId{Id: pointer.To(v["subnet_id"].(string))}) + } + + return pointer.To(props) +} + func flattenHDInsightComputeIsolationProperties(input *clusters.ComputeIsolationProperties) []interface{} { hostSku := "" enableComputeIsolation := false @@ -627,6 +687,45 @@ func flattenHDInsightsNetwork(input *clusters.NetworkProperties) []interface{} { } } +func flattenHDInsightPrivateLinkConfigurations(input *[]clusters.PrivateLinkConfiguration) []interface{} { + if input == nil { + return make([]interface{}, 0) + } + + v := pointer.From(input)[0] + ipConfig := v.Properties.IPConfigurations[0] + return []interface{}{ + map[string]interface{}{ + "name": v.Name, + "group_id": v.Properties.GroupId, + "ip_configuration": flattenHDInsightPrivateLinkConfigurationIpConfigurationProperties(&ipConfig), + }, + } +} +func flattenHDInsightPrivateLinkConfigurationIpConfigurationProperties(input *clusters.IPConfiguration) []interface{} { + if input == nil { + return make([]interface{}, 0) + } + + if input.Properties != nil { + return []interface{}{ + map[string]interface{}{ + "name": input.Name, + "primary": pointer.From(input.Properties.Primary), + "private_ip_allocation_method": pointer.From(input.Properties.PrivateIPAllocationMethod), + "private_ip_address": pointer.From(input.Properties.PrivateIPAddress), + "subnet_id": pointer.From(input.Properties.Subnet.Id), + }, + } + } + + return []interface{}{ + map[string]interface{}{ + "name": input.Name, + }, + } +} + func FlattenHDInsightsConfigurations(input map[string]string, d *pluginsdk.ResourceData) []interface{} { username := "" if v, exists := input["restAuthCredential.username"]; exists { @@ -860,6 +959,71 @@ func SchemaHDInsightsDiskEncryptionProperties() *pluginsdk.Schema { } } +func SchemaHDInsightPrivateLinkConfigurations() *pluginsdk.Schema { + return &pluginsdk.Schema{ + Type: pluginsdk.TypeList, + Optional: true, + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "name": { + Type: pluginsdk.TypeString, + Required: true, + }, + + "group_id": { + Type: pluginsdk.TypeString, + Required: true, + }, + + "ip_configuration": SchemaHDInsightPrivateLinkConfigurationIpConfiguration(), + }, + }, + } +} + +func SchemaHDInsightPrivateLinkConfigurationIpConfiguration() *pluginsdk.Schema { + return &pluginsdk.Schema{ + Type: pluginsdk.TypeList, + Required: true, + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "name": { + Type: pluginsdk.TypeString, + Required: true, + }, + + "primary": { + Type: pluginsdk.TypeBool, + Optional: true, + }, + + "private_ip_address": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: validation.IsIPAddress, + }, + + "private_ip_allocation_method": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{ + string(clusters.PrivateIPAllocationMethodDynamic), + string(clusters.PrivateIPAllocationMethodStatic), + }, false), + }, + + "subnet_id": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: commonids.ValidateSubnetID, + }, + }, + }, + } +} + func ExpandHDInsightsDiskEncryptionProperties(input []interface{}) (*clusters.DiskEncryptionProperties, error) { v := input[0].(map[string]interface{}) diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index b1c655dfed28..6219fe0a3a2f 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -99,6 +99,8 @@ The following arguments are supported: * `network` - (Optional) A `network` block as defined below. +* `private_link_configuration` - (Optional) A `private_link_configuration` block as defined below. + * `disk_encryption` - (Optional) One or more `disk_encryption` block as defined below. * `compute_isolation` - (Optional) A `compute_isolation` block as defined below. @@ -235,6 +237,30 @@ A `storage_account_gen2` block supports the following: --- +A `private_link_configuration` block supports the following: + +* `name` - (Required) The name of the private link configuration. + +* `group_id` - (Required) The ID of the private link service group. + +* `private_link_service_connection` - (Required) A `private_link_service_connection` block as defined below. + +--- + +A `private_link_service_connection` block supports the following: + +* `name` - (Required) The name of the private link service connection. + +* `primary` - (Optional) Indicates whether this IP configuration is primary. + +* `private_ip_allocation_method` - (Optional) The private IP allocation method. The only possible value now is `Dynamic`. + +* `private_ip_address` - (Optional) The private IP address of the IP configuration. + +* `subnet_id` - (Optional) The ID of the Subnet within the Virtual Network where the private link service connection should be provisioned within. + +--- + A `worker_node` block supports the following: * `username` - (Required) The Username of the local administrator for the Worker Nodes. Changing this forces a new resource to be created. diff --git a/website/docs/r/hdinsight_hbase_cluster.html.markdown b/website/docs/r/hdinsight_hbase_cluster.html.markdown index aecac3a4c410..54c1879a2401 100644 --- a/website/docs/r/hdinsight_hbase_cluster.html.markdown +++ b/website/docs/r/hdinsight_hbase_cluster.html.markdown @@ -99,6 +99,8 @@ The following arguments are supported: * `network` - (Optional) A `network` block as defined below. +* `private_link_configuration` - (Optional) A `private_link_configuration` block as defined below. + * `compute_isolation` - (Optional) A `compute_isolation` block as defined below. * `storage_account` - (Optional) One or more `storage_account` block as defined below. @@ -233,6 +235,30 @@ A `storage_account_gen2` block supports the following: --- +A `private_link_configuration` block supports the following: + +* `name` - (Required) The name of the private link configuration. + +* `group_id` - (Required) The ID of the private link service group. + +* `private_link_service_connection` - (Required) A `private_link_service_connection` block as defined below. + +--- + +A `private_link_service_connection` block supports the following: + +* `name` - (Required) The name of the private link service connection. + +* `primary` - (Optional) Indicates whether this IP configuration is primary. + +* `private_ip_allocation_method` - (Optional) The private IP allocation method. The only possible value now is `Dynamic`. + +* `private_ip_address` - (Optional) The private IP address of the IP configuration. + +* `subnet_id` - (Optional) The ID of the Subnet within the Virtual Network where the private link service connection should be provisioned within. + +--- + A `worker_node` block supports the following: * `script_actions` - (Optional) The script action which will run on the cluster. One or more `script_actions` blocks as defined above. diff --git a/website/docs/r/hdinsight_interactive_query_cluster.html.markdown b/website/docs/r/hdinsight_interactive_query_cluster.html.markdown index 9da8dcc0b8ca..0bcd622fd00c 100644 --- a/website/docs/r/hdinsight_interactive_query_cluster.html.markdown +++ b/website/docs/r/hdinsight_interactive_query_cluster.html.markdown @@ -101,6 +101,8 @@ The following arguments are supported: * `network` - (Optional) A `network` block as defined below. +* `private_link_configuration` - (Optional) A `private_link_configuration` block as defined below. + * `compute_isolation` - (Optional) A `compute_isolation` block as defined below. * `storage_account` - (Optional) One or more `storage_account` block as defined below. @@ -237,6 +239,30 @@ A `storage_account_gen2` block supports the following: --- +A `private_link_configuration` block supports the following: + +* `name` - (Required) The name of the private link configuration. + +* `group_id` - (Required) The ID of the private link service group. + +* `private_link_service_connection` - (Required) A `private_link_service_connection` block as defined below. + +--- + +A `private_link_service_connection` block supports the following: + +* `name` - (Required) The name of the private link service connection. + +* `primary` - (Optional) Indicates whether this IP configuration is primary. + +* `private_ip_allocation_method` - (Optional) The private IP allocation method. The only possible value now is `Dynamic`. + +* `private_ip_address` - (Optional) The private IP address of the IP configuration. + +* `subnet_id` - (Optional) The ID of the Subnet within the Virtual Network where the private link service connection should be provisioned within. + +--- + A `worker_node` block supports the following: * `script_actions` - (Optional) The script action which will run on the cluster. One or more `script_actions` blocks as defined above. diff --git a/website/docs/r/hdinsight_kafka_cluster.html.markdown b/website/docs/r/hdinsight_kafka_cluster.html.markdown index 573256da1d92..e2a425211a2c 100644 --- a/website/docs/r/hdinsight_kafka_cluster.html.markdown +++ b/website/docs/r/hdinsight_kafka_cluster.html.markdown @@ -98,6 +98,8 @@ The following arguments are supported: * `network` - (Optional) A `network` block as defined below. +* `private_link_configuration` - (Optional) A `private_link_configuration` block as defined below. + * `storage_account` - (Optional) One or more `storage_account` block as defined below. * `storage_account_gen2` - (Optional) A `storage_account_gen2` block as defined below. @@ -232,6 +234,30 @@ A `storage_account_gen2` block supports the following: --- +A `private_link_configuration` block supports the following: + +* `name` - (Required) The name of the private link configuration. + +* `group_id` - (Required) The ID of the private link service group. + +* `private_link_service_connection` - (Required) A `private_link_service_connection` block as defined below. + +--- + +A `private_link_service_connection` block supports the following: + +* `name` - (Required) The name of the private link service connection. + +* `primary` - (Optional) Indicates whether this IP configuration is primary. + +* `private_ip_allocation_method` - (Optional) The private IP allocation method. The only possible value now is `Dynamic`. + +* `private_ip_address` - (Optional) The private IP address of the IP configuration. + +* `subnet_id` - (Optional) The ID of the Subnet within the Virtual Network where the private link service connection should be provisioned within. + +--- + A `worker_node` block supports the following: * `script_actions` - (Optional) The script action which will run on the cluster. One or more `script_actions` blocks as defined below. diff --git a/website/docs/r/hdinsight_spark_cluster.html.markdown b/website/docs/r/hdinsight_spark_cluster.html.markdown index 42d7dd4c1e64..9a74d30bce3c 100644 --- a/website/docs/r/hdinsight_spark_cluster.html.markdown +++ b/website/docs/r/hdinsight_spark_cluster.html.markdown @@ -101,6 +101,8 @@ The following arguments are supported: * `network` - (Optional) A `network` block as defined below. +* `private_link_configuration` - (Optional) A `private_link_configuration` block as defined below. + * `compute_isolation` - (Optional) A `compute_isolation` block as defined below. * `storage_account` - (Optional) One or more `storage_account` block as defined below. @@ -235,6 +237,30 @@ A `storage_account_gen2` block supports the following: --- +A `private_link_configuration` block supports the following: + +* `name` - (Required) The name of the private link configuration. + +* `group_id` - (Required) The ID of the private link service group. + +* `private_link_service_connection` - (Required) A `private_link_service_connection` block as defined below. + +--- + +A `private_link_service_connection` block supports the following: + +* `name` - (Required) The name of the private link service connection. + +* `primary` - (Optional) Indicates whether this IP configuration is primary. + +* `private_ip_allocation_method` - (Optional) The private IP allocation method. The only possible value now is `Dynamic`. + +* `private_ip_address` - (Optional) The private IP address of the IP configuration. + +* `subnet_id` - (Optional) The ID of the Subnet within the Virtual Network where the private link service connection should be provisioned within. + +--- + A `worker_node` block supports the following: * `script_actions` - (Optional) The script action which will run on the cluster. One or more `script_actions` blocks as defined above. From a01dad5e526816e378fafcc1866242dba32baac5 Mon Sep 17 00:00:00 2001 From: Jiawei Tao Date: Tue, 16 Apr 2024 16:10:00 +0800 Subject: [PATCH 2/2] fix kafka cluster test --- .../hdinsight_hadoop_cluster_resource_test.go | 6 +----- .../hdinsight_kafka_cluster_resource_test.go | 11 ++++++----- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go index 42661e5c61cd..3f501b033931 100644 --- a/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go @@ -1795,11 +1795,7 @@ resource "azurerm_storage_container" "test" { func (HDInsightHadoopClusterResource) gen2template(data acceptance.TestData) string { return fmt.Sprintf(` provider "azurerm" { - features { - resource_group { - prevent_deletion_if_contains_resources = false - } - } + features {} } resource "azurerm_resource_group" "test" { diff --git a/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go index f40705b5ba4f..f0906245a4fe 100644 --- a/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_kafka_cluster_resource_test.go @@ -793,7 +793,7 @@ resource "azurerm_hdinsight_kafka_cluster" "test" { tier = "Standard" component_version { - hadoop = "3.1" + kafka = "2.1" } network { @@ -835,10 +835,11 @@ resource "azurerm_hdinsight_kafka_cluster" "test" { } worker_node { - vm_size = "Standard_D4_V2" - username = "acctestusrvm" - password = "AccTestvdSC4daf986!" - target_instance_count = 3 + vm_size = "Standard_D4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 3 + number_of_disks_per_node = 2 subnet_id = azurerm_subnet.test.id virtual_network_id = azurerm_virtual_network.test.id