From 85dbf30ae6c03e702f4ef7e99dc2aa8a3e296b7b Mon Sep 17 00:00:00 2001 From: Daniel Intskirveli Date: Wed, 16 Oct 2019 11:29:38 -0400 Subject: [PATCH] Support for launching HDInsight cluster with Data Lake Gen2 Filesystem --- azurerm/helpers/azure/hdinsight.go | 89 +++++++++++++++---- .../resource_arm_hdinsight_hadoop_cluster.go | 5 +- .../resource_arm_hdinsight_hbase_cluster.go | 5 +- ...arm_hdinsight_interactive_query_cluster.go | 5 +- .../resource_arm_hdinsight_kafka_cluster.go | 5 +- ...ource_arm_hdinsight_ml_services_cluster.go | 5 +- .../resource_arm_hdinsight_rserver_cluster.go | 5 +- .../resource_arm_hdinsight_spark_cluster.go | 5 +- .../resource_arm_hdinsight_storm_cluster.go | 5 +- .../r/hdinsight_hadoop_cluster.html.markdown | 10 ++- 10 files changed, 105 insertions(+), 34 deletions(-) diff --git a/azurerm/helpers/azure/hdinsight.go b/azurerm/helpers/azure/hdinsight.go index 03aebe9e8357d..88fc61c235604 100644 --- a/azurerm/helpers/azure/hdinsight.go +++ b/azurerm/helpers/azure/hdinsight.go @@ -157,14 +157,32 @@ func SchemaHDInsightsStorageAccounts() *schema.Schema { Schema: map[string]*schema.Schema{ "storage_account_key": { Type: schema.TypeString, - Required: true, + Optional: true, ForceNew: true, Sensitive: true, ValidateFunc: validate.NoEmptyStrings, }, "storage_container_id": { Type: schema.TypeString, - Required: true, + Optional: true, + ForceNew: true, + ValidateFunc: validate.NoEmptyStrings, + }, + "filesystem_id": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validate.NoEmptyStrings, + }, + "storage_resource_id": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + ValidateFunc: validate.NoEmptyStrings, + }, + "managed_identity_resource_id": { + Type: schema.TypeString, + Optional: true, ForceNew: true, ValidateFunc: validate.NoEmptyStrings, }, @@ -178,32 +196,71 @@ func SchemaHDInsightsStorageAccounts() *schema.Schema { } } -func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageAccount, error) { +// ExpandHDInsightsStorageAccounts returns an array of StorageAccount structs, as well as a ClusterIdentity +// populated with any managed identities required for accessing Data Lake Gen2 storage. +func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageAccount, *hdinsight.ClusterIdentity, error) { results := make([]hdinsight.StorageAccount, 0) + var clusterIndentity *hdinsight.ClusterIdentity + for _, vs := range input { v := vs.(map[string]interface{}) storageAccountKey := v["storage_account_key"].(string) - storageContainerId := v["storage_container_id"].(string) + + storageContainerID := v["storage_container_id"].(string) + + fileSystemID := v["filesystem_id"].(string) + storageResourceID := v["storage_resource_id"].(string) + managedIdentityResourceID := v["managed_identity_resource_id"].(string) + isDefault := v["is_default"].(bool) - // https://foo.blob.core.windows.net/example - uri, err := url.Parse(storageContainerId) - if err != nil { - return nil, fmt.Errorf("Error parsing %q: %s", storageContainerId, err) - } + if fileSystemID == "" && storageResourceID == "" && managedIdentityResourceID == "" && storageContainerID != "" && storageAccountKey != "" { + uri, err := url.Parse(storageContainerID) + if err != nil { + return nil, nil, fmt.Errorf("Error parsing %q: %s", storageContainerID, err) + } + + result := hdinsight.StorageAccount{ + Name: utils.String(uri.Host), + Container: utils.String(strings.TrimPrefix(uri.Path, "/")), + Key: utils.String(storageAccountKey), + IsDefault: utils.Bool(isDefault), + } + results = append(results, result) + } else if fileSystemID != "" && storageResourceID != "" && managedIdentityResourceID != "" && storageContainerID == "" && storageAccountKey == "" { - result := hdinsight.StorageAccount{ - Name: utils.String(uri.Host), - Container: utils.String(strings.TrimPrefix(uri.Path, "/")), - Key: utils.String(storageAccountKey), - IsDefault: utils.Bool(isDefault), + uri, err := url.Parse(fileSystemID) + if err != nil { + return nil, nil, fmt.Errorf("Error parsing %q: %s", storageContainerID, err) + } + + if clusterIndentity == nil { + clusterIndentity = &hdinsight.ClusterIdentity{ + Type: hdinsight.UserAssigned, + UserAssignedIdentities: make(map[string]*hdinsight.ClusterIdentityUserAssignedIdentitiesValue), + } + } + + // ... API doesn't seem to require client_id or principal_id, so pass in an empty ClusterIdentityUserAssignedIdentitiesValue + clusterIndentity.UserAssignedIdentities[managedIdentityResourceID] = &hdinsight.ClusterIdentityUserAssignedIdentitiesValue{} + + result := hdinsight.StorageAccount{ + Name: utils.String(uri.Host), // https://storageaccountname.dfs.core.windows.net/filesystemname -> storageaccountname.dfs.core.windows.net + ResourceID: utils.String(storageResourceID), + FileSystem: utils.String(uri.Path[1:]), // https://storageaccountname.dfs.core.windows.net/filesystemname -> filesystemname + MsiResourceID: utils.String(managedIdentityResourceID), + IsDefault: utils.Bool(isDefault), + } + results = append(results, result) + } else { + return nil, nil, fmt.Errorf(`specify either storage_container_id AND storage_account_key (for WASB blob storage), ` + + `or filesystem_id AND storage_resource_id AND managed_identity_resource_id (for ata Lake Storage Gen 2)`) } - results = append(results, result) } - return &results, nil + return &results, clusterIndentity, nil } type HDInsightNodeDefinition struct { diff --git a/azurerm/resource_arm_hdinsight_hadoop_cluster.go b/azurerm/resource_arm_hdinsight_hadoop_cluster.go index 453d1fcad6d0c..9dcbd1ed242e5 100644 --- a/azurerm/resource_arm_hdinsight_hadoop_cluster.go +++ b/azurerm/resource_arm_hdinsight_hadoop_cluster.go @@ -138,7 +138,7 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -185,7 +185,8 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_hbase_cluster.go b/azurerm/resource_arm_hdinsight_hbase_cluster.go index 2c0a90f2b4d21..040c5c566a11f 100644 --- a/azurerm/resource_arm_hdinsight_hbase_cluster.go +++ b/azurerm/resource_arm_hdinsight_hbase_cluster.go @@ -136,7 +136,7 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -183,7 +183,8 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_interactive_query_cluster.go b/azurerm/resource_arm_hdinsight_interactive_query_cluster.go index 28a975993cb71..fde8533e5713a 100644 --- a/azurerm/resource_arm_hdinsight_interactive_query_cluster.go +++ b/azurerm/resource_arm_hdinsight_interactive_query_cluster.go @@ -136,7 +136,7 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -183,7 +183,8 @@ func resourceArmHDInsightInteractiveQueryClusterCreate(d *schema.ResourceData, m Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_kafka_cluster.go b/azurerm/resource_arm_hdinsight_kafka_cluster.go index d5a7f69baca0a..de7cefb9e45d6 100644 --- a/azurerm/resource_arm_hdinsight_kafka_cluster.go +++ b/azurerm/resource_arm_hdinsight_kafka_cluster.go @@ -137,7 +137,7 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -184,7 +184,8 @@ func resourceArmHDInsightKafkaClusterCreate(d *schema.ResourceData, meta interfa Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_ml_services_cluster.go b/azurerm/resource_arm_hdinsight_ml_services_cluster.go index 2f9ba98a4044f..d89be281b9cd6 100644 --- a/azurerm/resource_arm_hdinsight_ml_services_cluster.go +++ b/azurerm/resource_arm_hdinsight_ml_services_cluster.go @@ -153,7 +153,7 @@ func resourceArmHDInsightMLServicesClusterCreate(d *schema.ResourceData, meta in gateway := expandHDInsightsMLServicesConfigurations(gatewayRaw, rStudio) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -200,7 +200,8 @@ func resourceArmHDInsightMLServicesClusterCreate(d *schema.ResourceData, meta in Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_rserver_cluster.go b/azurerm/resource_arm_hdinsight_rserver_cluster.go index 8f67f44c5d7d5..04460a624a5a1 100644 --- a/azurerm/resource_arm_hdinsight_rserver_cluster.go +++ b/azurerm/resource_arm_hdinsight_rserver_cluster.go @@ -153,7 +153,7 @@ func resourceArmHDInsightRServerClusterCreate(d *schema.ResourceData, meta inter gateway := expandHDInsightsRServerConfigurations(gatewayRaw, rStudio) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -200,7 +200,8 @@ func resourceArmHDInsightRServerClusterCreate(d *schema.ResourceData, meta inter Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_spark_cluster.go b/azurerm/resource_arm_hdinsight_spark_cluster.go index 5d44a2fca2ab8..47ff63ad614aa 100644 --- a/azurerm/resource_arm_hdinsight_spark_cluster.go +++ b/azurerm/resource_arm_hdinsight_spark_cluster.go @@ -136,7 +136,7 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -183,7 +183,8 @@ func resourceArmHDInsightSparkClusterCreate(d *schema.ResourceData, meta interfa Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/azurerm/resource_arm_hdinsight_storm_cluster.go b/azurerm/resource_arm_hdinsight_storm_cluster.go index 922fe3f9f237a..df2e422dbc82f 100644 --- a/azurerm/resource_arm_hdinsight_storm_cluster.go +++ b/azurerm/resource_arm_hdinsight_storm_cluster.go @@ -137,7 +137,7 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw) storageAccountsRaw := d.Get("storage_account").([]interface{}) - storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) + storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw) if err != nil { return fmt.Errorf("Error expanding `storage_account`: %s", err) } @@ -184,7 +184,8 @@ func resourceArmHDInsightStormClusterCreate(d *schema.ResourceData, meta interfa Roles: roles, }, }, - Tags: tags.Expand(t), + Tags: tags.Expand(t), + Identity: identity, } future, err := client.Create(ctx, resourceGroup, name, params) if err != nil { diff --git a/website/docs/r/hdinsight_hadoop_cluster.html.markdown b/website/docs/r/hdinsight_hadoop_cluster.html.markdown index c677ace1e5ab0..c3111da4f359e 100644 --- a/website/docs/r/hdinsight_hadoop_cluster.html.markdown +++ b/website/docs/r/hdinsight_hadoop_cluster.html.markdown @@ -161,12 +161,18 @@ A `storage_account` block supports the following: -> **NOTE:** One of the `storage_account` blocks must be marked as the default. -* `storage_account_key` - (Required) The Access Key which should be used to connect to the Storage Account. Changing this forces a new resource to be created. +* `storage_account_key` - (Required for Blob storage) The Access Key which should be used to connect to the Storage Account. Changing this forces a new resource to be created. -* `storage_container_id` - (Required) The ID of the Storage Container. Changing this forces a new resource to be created. +* `storage_container_id` - (Required for Blob storage) The ID of the Storage Container. Changing this forces a new resource to be created. -> **NOTE:** This can be obtained from the `id` of the `azurerm_storage_container` resource. +* `storage_resource_id` - (Required for Gen2 storage) The resource ID of the Storage Account. Changing this forces a new resource to be created. + +* `filesystem_id` - (Required for Gen2 storage) The ID of the Gen2 filesystem. See `azurerm_storage_data_lake_gen2_filesystem`. Changing this forces a new resource to be created. + +* `managed_identity_resource_id` - (Required for Gen2 storage) The ID managed identity for access to the Gen2 filesystem. Changing this forces a new resource to be created. + --- A `worker_node` block supports the following: