Skip to content

Commit

Permalink
Merge pull request #6969 from kosinsky/external_mtastore
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackofallops
jackofallops authored Jul 8, 2020
2 parents c69912d + bd8bef0 commit a1519f1
Show file tree
Hide file tree
Showing 27 changed files with 1,762 additions and 748 deletions.
17 changes: 17 additions & 0 deletions azurerm/helpers/azure/hdinsight.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,23 @@ func SchemaHDInsightsExternalMetastore() *schema.Schema {
}
}

func SchemaHDInsightsExternalMetastores() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": SchemaHDInsightsExternalMetastore(),

"oozie": SchemaHDInsightsExternalMetastore(),

"ambari": SchemaHDInsightsExternalMetastore(),
},
},
}
}

func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

Expand Down
15 changes: 1 addition & 14 deletions azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,7 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": azure.SchemaHDInsightsExternalMetastore(),

"oozie": azure.SchemaHDInsightsExternalMetastore(),

"ambari": azure.SchemaHDInsightsExternalMetastore(),
},
},
},
"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightHBaseCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightHBaseComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("HBase"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

hbaseRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightInteractiveQueryCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
componentVersions := expandHDInsightInteractiveQueryComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("INTERACTIVEHIVE"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
return fmt.Errorf("Error retrieving HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

interactiveQueryRoles := hdInsightRoleDefinition{
Expand Down
26 changes: 21 additions & 5 deletions azurerm/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ func resourceArmHDInsightKafkaCluster() *schema.Resource {

"tls_min_version": azure.SchemaHDInsightTls(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"component_version": {
Type: schema.TypeList,
Required: true,
Expand Down Expand Up @@ -139,7 +141,13 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightKafkaComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -182,7 +190,7 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Kafka"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -242,9 +250,15 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Kafka Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Kafka Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
Expand All @@ -264,9 +278,11 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

kafkaRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_spark_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightSparkCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightSparkComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Spark"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

sparkRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_storm_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightStormCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"roles": {
Expand Down Expand Up @@ -136,7 +138,13 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightStormComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, nil)
Expand Down Expand Up @@ -178,7 +186,7 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Storm"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -238,11 +246,17 @@ func resourceArmHDInsightStormClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -260,9 +274,11 @@ func resourceArmHDInsightStormClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

stormRoles := hdInsightRoleDefinition{
Expand Down
Loading

0 comments on commit a1519f1

Please sign in to comment.