Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

External metastores support for most HDInsight resources #6969

Merged
merged 20 commits into from
Jul 8, 2020
Merged

External metastores support for most HDInsight resources #6969

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f7bfeaf
r/postgresql_server: requiring the use of `ssl_enforcement` since `d.…
tombuildsstuff Jun 12, 2020
6cd20f7
`make lint`
tombuildsstuff Jun 12, 2020
8898d9b
External metastores for HBase
kosinsky May 15, 2020
c68b354
External metastores for Interactive Hive
kosinsky May 15, 2020
3d7612b
External metastores for Kafka
kosinsky May 15, 2020
456aeba
External metastores for MLServices
kosinsky May 15, 2020
b90be9a
External metastores for RServer
kosinsky May 15, 2020
f3ddcb2
EXternal metastores for Spark
kosinsky May 15, 2020
711f683
Error message fixes
kosinsky May 15, 2020
b0357be
External metastore for Storm
kosinsky May 15, 2020
9a7db8a
Fix for msservices / rserver
kosinsky May 16, 2020
a36c73b
Revert "Fix for msservices / rserver"
kosinsky May 16, 2020
b8943f7
Revert "External metastores for RServer"
kosinsky May 16, 2020
3545c58
Revert "External metastores for MLServices"
kosinsky May 16, 2020
bd5c6d3
Documentation for external metastores
kosinsky May 17, 2020
5e2c5a9
Remove GetOkExists
kosinsky Jun 12, 2020
41abfd7
Fix typo in hbase
kosinsky Jun 12, 2020
69c1fba
Get rid of GetOkExists in all hdinsight resources
kosinsky Jun 12, 2020
a01ee6f
merge conflict resolution and minor change for metastoreraw
jackofallops Jul 6, 2020
bd8bef0
deps
jackofallops Jul 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions azurerm/helpers/azure/hdinsight.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,23 @@ func SchemaHDInsightsExternalMetastore() *schema.Schema {
}
}

func SchemaHDInsightsExternalMetastores() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": SchemaHDInsightsExternalMetastore(),

"oozie": SchemaHDInsightsExternalMetastore(),

"ambari": SchemaHDInsightsExternalMetastore(),
},
},
}
}

func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

Expand Down
15 changes: 1 addition & 14 deletions azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,7 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": azure.SchemaHDInsightsExternalMetastore(),

"oozie": azure.SchemaHDInsightsExternalMetastore(),

"ambari": azure.SchemaHDInsightsExternalMetastore(),
},
},
},
"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightHBaseCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightHBaseComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("HBase"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight HBase Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightHBaseClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

hbaseRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightInteractiveQueryCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
componentVersions := expandHDInsightInteractiveQueryComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("INTERACTIVEHIVE"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
return fmt.Errorf("Error retrieving HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Interactive Query Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightInteractiveQueryClusterRead(d *schema.ResourceData, met
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

interactiveQueryRoles := hdInsightRoleDefinition{
Expand Down
26 changes: 21 additions & 5 deletions azurerm/internal/services/hdinsight/hdinsight_kafka_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ func resourceArmHDInsightKafkaCluster() *schema.Resource {

"tls_min_version": azure.SchemaHDInsightTls(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"component_version": {
Type: schema.TypeList,
Required: true,
Expand Down Expand Up @@ -139,7 +141,13 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightKafkaComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -182,7 +190,7 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Kafka"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -242,9 +250,15 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Kafka Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Kafka Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("Error retrieving Configuration for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Hadoop Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
Expand All @@ -264,9 +278,11 @@ func resourceArmHDInsightKafkaClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

kafkaRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_spark_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightSparkCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
Expand Down Expand Up @@ -138,7 +140,13 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightSparkComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
Expand Down Expand Up @@ -181,7 +189,7 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Spark"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -241,11 +249,17 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Spark Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -263,9 +277,11 @@ func resourceArmHDInsightSparkClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

sparkRoles := hdInsightRoleDefinition{
Expand Down
24 changes: 20 additions & 4 deletions azurerm/internal/services/hdinsight/hdinsight_storm_cluster_resource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ func resourceArmHDInsightStormCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"roles": {
Expand Down Expand Up @@ -136,7 +138,13 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa
componentVersions := expandHDInsightStormComponentVersion(componentVersionsRaw)

gatewayRaw := d.Get("gateway").([]interface{})
gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, nil)
Expand Down Expand Up @@ -178,7 +186,7 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("Storm"),
ComponentVersion: componentVersions,
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -238,11 +246,17 @@ func resourceArmHDInsightStormClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error retrieving HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Configuration for HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight Storm Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
d.Set("resource_group_name", resourceGroup)
if location := resp.Location; location != nil {
Expand All @@ -260,9 +274,11 @@ func resourceArmHDInsightStormClusterRead(d *schema.ResourceData, meta interface
return fmt.Errorf("Error flattening `component_version`: %+v", err)
}

if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)
}

stormRoles := hdInsightRoleDefinition{
Expand Down
Loading