azurerm_hdinsight_* - support HDInsight clusters with Data Lake Gen2 Filesystems

azurerm/helpers/azure/hdinsight.go

@@ -153,7 +153,7 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 func SchemaHDInsightsStorageAccounts() *schema.Schema {
 	return &schema.Schema{
 		Type:     schema.TypeList,
-		Required: true,
+		Optional: true,
 		Elem: &schema.Resource{
 			Schema: map[string]*schema.Schema{
 				"storage_account_key": {
@@ -179,20 +179,60 @@ func SchemaHDInsightsStorageAccounts() *schema.Schema {
 	}
 }
 
-func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageAccount, error) {
+func SchemaHDInsightsGen2StorageAccounts() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		// HDInsight doesn't seem to allow adding more than one gen2 cluster right now.
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"storage_resource_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: ValidateResourceID,
+				},
+				"filesystem_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: validate.NoEmptyStrings,
+				},
+				"managed_identity_resource_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: ValidateResourceID,
+				},
+				"is_default": {
+					Type:     schema.TypeBool,
+					Required: true,
+					ForceNew: true,
+				},
+			},
+		},
+	}
+}
+
+// ExpandHDInsightsStorageAccounts returns an array of StorageAccount structs, as well as a ClusterIdentity
+// populated with any managed identities required for accessing Data Lake Gen2 storage.
+func ExpandHDInsightsStorageAccounts(storageAccounts []interface{}, gen2storageAccounts []interface{}) (*[]hdinsight.StorageAccount, *hdinsight.ClusterIdentity, error) {
 	results := make([]hdinsight.StorageAccount, 0)
 
-	for _, vs := range input {
+	var clusterIndentity *hdinsight.ClusterIdentity
+
+	for _, vs := range storageAccounts {
 		v := vs.(map[string]interface{})
 
 		storageAccountKey := v["storage_account_key"].(string)
-		storageContainerId := v["storage_container_id"].(string)
+		storageContainerID := v["storage_container_id"].(string)
 		isDefault := v["is_default"].(bool)
 
-		// https://foo.blob.core.windows.net/example
-		uri, err := url.Parse(storageContainerId)
+		uri, err := url.Parse(storageContainerID)
+
 		if err != nil {
-			return nil, fmt.Errorf("Error parsing %q: %s", storageContainerId, err)
+			return nil, nil, fmt.Errorf("Error parsing %q: %s", storageContainerID, err)
 		}
 
 		result := hdinsight.StorageAccount{
@@ -204,7 +244,41 @@ func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageA
 		results = append(results, result)
 	}
 
-	return &results, nil
+	for _, vs := range gen2storageAccounts {
+		v := vs.(map[string]interface{})
+
+		fileSystemID := v["filesystem_id"].(string)
+		storageResourceID := v["storage_resource_id"].(string)
+		managedIdentityResourceID := v["managed_identity_resource_id"].(string)
+
+		isDefault := v["is_default"].(bool)
+
+		uri, err := url.Parse(fileSystemID)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Error parsing %q: %s", fileSystemID, err)
+		}
+
+		if clusterIndentity == nil {
+			clusterIndentity = &hdinsight.ClusterIdentity{
+				Type:                   hdinsight.UserAssigned,
+				UserAssignedIdentities: make(map[string]*hdinsight.ClusterIdentityUserAssignedIdentitiesValue),
+			}
+		}
+
+		// ... API doesn't seem to require client_id or principal_id, so pass in an empty ClusterIdentityUserAssignedIdentitiesValue
+		clusterIndentity.UserAssignedIdentities[managedIdentityResourceID] = &hdinsight.ClusterIdentityUserAssignedIdentitiesValue{}
+
+		result := hdinsight.StorageAccount{
+			Name:          utils.String(uri.Host), // https://storageaccountname.dfs.core.windows.net/filesystemname -> storageaccountname.dfs.core.windows.net
+			ResourceID:    utils.String(storageResourceID),
+			FileSystem:    utils.String(uri.Path[1:]), // https://storageaccountname.dfs.core.windows.net/filesystemname -> filesystemname
+			MsiResourceID: utils.String(managedIdentityResourceID),
+			IsDefault:     utils.Bool(isDefault),
+		}
+		results = append(results, result)
+	}
+
+	return &results, clusterIndentity, nil
 }
 
 type HDInsightNodeDefinition struct {

azurerm/resource_arm_hdinsight_hadoop_cluster.go

@@ -93,6 +93,8 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
 
 			"storage_account": azure.SchemaHDInsightsStorageAccounts(),
 
+			"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
+
 			"roles": {
 				Type:     schema.TypeList,
 				Required: true,
@@ -184,7 +186,8 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 	gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
 
 	storageAccountsRaw := d.Get("storage_account").([]interface{})
-	storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw)
+	storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
+	storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, storageAccountsGen2Raw)
 	if err != nil {
 		return fmt.Errorf("Error expanding `storage_account`: %s", err)
 	}
@@ -231,7 +234,8 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 				Roles: roles,
 			},
 		},
-		Tags: tags.Expand(t),
+		Tags:     tags.Expand(t),
+		Identity: identity,
 	}
 	future, err := client.Create(ctx, resourceGroup, name, params)
 	if err != nil {