azurerm_hdinsight_* - support HDInsight clusters with Data Lake Gen2 Filesystems

azurerm/helpers/azure/hdinsight.go

@@ -153,7 +153,7 @@ func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
 func SchemaHDInsightsStorageAccounts() *schema.Schema {
 	return &schema.Schema{
 		Type:     schema.TypeList,
-		Required: true,
+		Optional: true,
 		Elem: &schema.Resource{
 			Schema: map[string]*schema.Schema{
 				"storage_account_key": {
@@ -179,20 +179,60 @@ func SchemaHDInsightsStorageAccounts() *schema.Schema {
 	}
 }
 
-func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageAccount, error) {
+func SchemaHDInsightsGen2StorageAccounts() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		// HDInsight doesn't seem to allow adding more than one gen2 cluster right now.
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"storage_resource_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: validate.NoEmptyStrings,
+				},
+				"filesystem_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: validate.NoEmptyStrings,
+				},
+				"managed_identity_resource_id": {
+					Type:         schema.TypeString,
+					Required:     true,
+					ForceNew:     true,
+					ValidateFunc: validate.NoEmptyStrings,
+				},
+				"is_default": {
+					Type:     schema.TypeBool,
+					Required: true,
+					ForceNew: true,
+				},
+			},
+		},
+	}
+}
+
+// ExpandHDInsightsStorageAccounts returns an array of StorageAccount structs, as well as a ClusterIdentity
+// populated with any managed identities required for accessing Data Lake Gen2 storage.
+func ExpandHDInsightsStorageAccounts(storageAccounts []interface{}, gen2storageAccounts []interface{}) (*[]hdinsight.StorageAccount, *hdinsight.ClusterIdentity, error) {
 	results := make([]hdinsight.StorageAccount, 0)
 
-	for _, vs := range input {
+	var clusterIndentity *hdinsight.ClusterIdentity
+
+	for _, vs := range storageAccounts {
 		v := vs.(map[string]interface{})
 
 		storageAccountKey := v["storage_account_key"].(string)
-		storageContainerId := v["storage_container_id"].(string)
+		storageContainerID := v["storage_container_id"].(string)
 		isDefault := v["is_default"].(bool)
 
-		// https://foo.blob.core.windows.net/example
-		uri, err := url.Parse(storageContainerId)
+		uri, err := url.Parse(storageContainerID)
+
 		if err != nil {
-			return nil, fmt.Errorf("Error parsing %q: %s", storageContainerId, err)
+			return nil, nil, fmt.Errorf("Error parsing %q: %s", storageContainerID, err)
 		}
 
 		result := hdinsight.StorageAccount{
@@ -204,7 +244,41 @@ func ExpandHDInsightsStorageAccounts(input []interface{}) (*[]hdinsight.StorageA
 		results = append(results, result)
 	}
 
-	return &results, nil
+	for _, vs := range gen2storageAccounts {
+		v := vs.(map[string]interface{})
+
+		fileSystemID := v["filesystem_id"].(string)
+		storageResourceID := v["storage_resource_id"].(string)
+		managedIdentityResourceID := v["managed_identity_resource_id"].(string)
+
+		isDefault := v["is_default"].(bool)
+
+		uri, err := url.Parse(fileSystemID)
+		if err != nil {
+			return nil, nil, fmt.Errorf("Error parsing %q: %s", fileSystemID, err)
+		}
+
+		if clusterIndentity == nil {
+			clusterIndentity = &hdinsight.ClusterIdentity{
+				Type:                   hdinsight.UserAssigned,
+				UserAssignedIdentities: make(map[string]*hdinsight.ClusterIdentityUserAssignedIdentitiesValue),
+			}
+		}
+
+		// ... API doesn't seem to require client_id or principal_id, so pass in an empty ClusterIdentityUserAssignedIdentitiesValue
+		clusterIndentity.UserAssignedIdentities[managedIdentityResourceID] = &hdinsight.ClusterIdentityUserAssignedIdentitiesValue{}
+
+		result := hdinsight.StorageAccount{
+			Name:          utils.String(uri.Host), // https://storageaccountname.dfs.core.windows.net/filesystemname -> storageaccountname.dfs.core.windows.net
+			ResourceID:    utils.String(storageResourceID),
+			FileSystem:    utils.String(uri.Path[1:]), // https://storageaccountname.dfs.core.windows.net/filesystemname -> filesystemname
+			MsiResourceID: utils.String(managedIdentityResourceID),
+			IsDefault:     utils.Bool(isDefault),
+		}
+		results = append(results, result)
+	}
+
+	return &results, clusterIndentity, nil
 }
 
 type HDInsightNodeDefinition struct {

azurerm/resource_arm_hdinsight_hadoop_cluster.go

@@ -93,6 +93,8 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {
 
 			"storage_account": azure.SchemaHDInsightsStorageAccounts(),
 
+			"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
+
 			"roles": {
 				Type:     schema.TypeList,
 				Required: true,
@@ -184,7 +186,8 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 	gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
 
 	storageAccountsRaw := d.Get("storage_account").([]interface{})
-	storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw)
+	storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
+	storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, storageAccountsGen2Raw)
 	if err != nil {
 		return fmt.Errorf("Error expanding `storage_account`: %s", err)
 	}
@@ -231,7 +234,8 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
 				Roles: roles,
 			},
 		},
-		Tags: tags.Expand(t),
+		Tags:     tags.Expand(t),
+		Identity: identity,
 	}
 	future, err := client.Create(ctx, resourceGroup, name, params)
 	if err != nil {

azurerm/resource_arm_hdinsight_hadoop_cluster_test.go

@@ -374,6 +374,43 @@ func TestAccAzureRMHDInsightHadoopCluster_addEdgeNodeBasic(t *testing.T) {
 	})
 }
 
+func TestAccAzureRMHDInsightHadoopCluster_gen2storage(t *testing.T) {
+	resourceName := "azurerm_hdinsight_hadoop_cluster.test"
+	ri := tf.AccRandTimeInt()
+	rs := strings.ToLower(acctest.RandString(11))
+	location := testLocation()
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testCheckAzureRMHDInsightClusterDestroy("azurerm_hdinsight_hadoop_cluster"),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccAzureRMHDInsightHadoopCluster_gen2storage(ri, rs, location),
+				Check: resource.ComposeTestCheckFunc(
+					testCheckAzureRMHDInsightClusterExists(resourceName),
+					resource.TestCheckResourceAttrSet(resourceName, "https_endpoint"),
+					resource.TestCheckResourceAttrSet(resourceName, "ssh_endpoint"),
+				),
+			},
+			{
+				ResourceName:      resourceName,
+				ImportState:       true,
+				ImportStateVerify: true,
+				ImportStateVerifyIgnore: []string{
+					"roles.0.head_node.0.password",
+					"roles.0.head_node.0.vm_size",
+					"roles.0.worker_node.0.password",
+					"roles.0.worker_node.0.vm_size",
+					"roles.0.zookeeper_node.0.password",
+					"roles.0.zookeeper_node.0.vm_size",
+					"storage_account",
+				},
+			},
+		},
+	})
+}
+
 func testAccAzureRMHDInsightHadoopCluster_basic(rInt int, rString string, location string) string {
 	template := testAccAzureRMHDInsightHadoopCluster_template(rInt, rString, location)
 	return fmt.Sprintf(`
@@ -766,6 +803,54 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" {
 `, template, rInt, numEdgeNodes, instanceType)
 }
 
+func testAccAzureRMHDInsightHadoopCluster_gen2storage(rInt int, rString string, location string) string {
+	template := testAccAzureRMHDInsightHadoopCluster_gen2template(rInt, rString, location)
+	return fmt.Sprintf(`
+%s
+resource "azurerm_hdinsight_hadoop_cluster" "test" {
+  depends_on = [azurerm_role_assignment.test]
+
+  name                = "acctesthdi-%d"
+  resource_group_name = "${azurerm_resource_group.test.name}"
+  location            = "${azurerm_resource_group.test.location}"
+  cluster_version     = "3.6"
+  tier                = "Standard"
+  component_version {
+    hadoop = "2.7"
+  }
+  gateway {
+    enabled  = true
+    username = "acctestusrgw"
+    password = "TerrAform123!"
+  }
+  storage_account_gen2 {
+    storage_resource_id          = azurerm_storage_account.gen2test.id
+    filesystem_id                = azurerm_storage_data_lake_gen2_filesystem.gen2test.id
+    managed_identity_resource_id = azurerm_user_assigned_identity.test.id
+    is_default                   = true
+  }
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+    worker_node {
+      vm_size               = "Standard_D4_V2"
+      username              = "acctestusrvm"
+      password              = "AccTestvdSC4daf986!"
+      target_instance_count = 2
+    }
+    zookeeper_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+  }
+}
+`, template, rInt)
+}
+
 func testAccAzureRMHDInsightHadoopCluster_template(rInt int, rString string, location string) string {
 	return fmt.Sprintf(`
 resource "azurerm_resource_group" "test" {
@@ -783,9 +868,50 @@ resource "azurerm_storage_account" "test" {
 
 resource "azurerm_storage_container" "test" {
   name                  = "acctest"
-  resource_group_name   = "${azurerm_resource_group.test.name}"
   storage_account_name  = "${azurerm_storage_account.test.name}"
   container_access_type = "private"
 }
+
+`, rInt, location, rString)
+}
+
+func testAccAzureRMHDInsightHadoopCluster_gen2template(rInt int, rString string, location string) string {
+	return fmt.Sprintf(`
+resource "azurerm_resource_group" "test" {
+  name     = "acctestRG-%d"
+  location = "%s"
+}
+
+resource "azurerm_storage_account" "gen2test" {
+  name                     = "accgen2test%s"
+  resource_group_name      = azurerm_resource_group.test.name
+  location                 = azurerm_resource_group.test.location
+  account_kind             = "StorageV2"
+  account_tier             = "Standard"
+  account_replication_type = "LRS"
+  is_hns_enabled           = true
+}
+
+resource "azurerm_storage_data_lake_gen2_filesystem" "gen2test" {
+  name               = "acctest"
+  storage_account_id = azurerm_storage_account.gen2test.id
+}
+
+resource "azurerm_user_assigned_identity" "test" {
+  resource_group_name = "${azurerm_resource_group.test.name}"
+  location            = "${azurerm_resource_group.test.location}"
+
+  name = "test-identity"
+}
+
+data "azurerm_subscription" "primary" {}
+
+
+resource "azurerm_role_assignment" "test" {
+  scope                = "${data.azurerm_subscription.primary.id}"
+  role_definition_name = "Storage Blob Data Owner"
+  principal_id         = "${azurerm_user_assigned_identity.test.principal_id}"
+}
+
 `, rInt, location, rString)
 }

azurerm/resource_arm_hdinsight_hbase_cluster.go

@@ -87,6 +87,8 @@ func resourceArmHDInsightHBaseCluster() *schema.Resource {
 
 			"storage_account": azure.SchemaHDInsightsStorageAccounts(),
 
+			"storage_account_gen2": azure.SchemaHDInsightsGen2StorageAccounts(),
+
 			"roles": {
 				Type:     schema.TypeList,
 				Required: true,
@@ -136,7 +138,8 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
 	gateway := azure.ExpandHDInsightsConfigurations(gatewayRaw)
 
 	storageAccountsRaw := d.Get("storage_account").([]interface{})
-	storageAccounts, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw)
+	storageAccountsGen2Raw := d.Get("storage_account_gen2").([]interface{})
+	storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, storageAccountsGen2Raw)
 	if err != nil {
 		return fmt.Errorf("Error expanding `storage_account`: %s", err)
 	}
@@ -183,7 +186,8 @@ func resourceArmHDInsightHBaseClusterCreate(d *schema.ResourceData, meta interfa
 				Roles: roles,
 			},
 		},
-		Tags: tags.Expand(t),
+		Tags:     tags.Expand(t),
+		Identity: identity,
 	}
 	future, err := client.Create(ctx, resourceGroup, name, params)
 	if err != nil {