Skip to content

Commit

Permalink
External metastores for RServer
Browse files Browse the repository at this point in the history
  • Loading branch information
@kosinsky
kosinsky committed May 15, 2020
1 parent 30536e7 commit a4166b0
Show file tree
Hide file tree
Showing 2 changed files with 321 additions and 9 deletions.
35 changes: 26 additions & 9 deletions azurerm/internal/services/hdinsight/hdinsight_rserver_cluster_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ func resourceArmHDInsightRServerCluster() *schema.Resource {
ForceNew: true,
},

"metastores": azure.SchemaHDInsightsExternalMetastores(),

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

"roles": {
Expand Down Expand Up @@ -154,7 +156,14 @@ func resourceArmHDInsightRServerClusterCreate(d *schema.ResourceData, meta inter

gatewayRaw := d.Get("gateway").([]interface{})
rStudio := d.Get("rstudio").(bool)
gateway := expandHDInsightsRServerConfigurations(gatewayRaw, rStudio)
configurations := expandHDInsightsMLServicesConfigurations(gatewayRaw, rStudio)

if metastoresRaw, ok := d.GetOkExists("metastores"); ok {
metastores := expandHDInsightsMetastore(metastoresRaw.([]interface{}))
for k, v := range metastores {
configurations[k] = v
}
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
storageAccounts, identity, err := azure.ExpandHDInsightsStorageAccounts(storageAccountsRaw, nil)
Expand Down Expand Up @@ -196,7 +205,7 @@ func resourceArmHDInsightRServerClusterCreate(d *schema.ResourceData, meta inter
MinSupportedTLSVersion: utils.String(tls),
ClusterDefinition: &hdinsight.ClusterDefinition{
Kind: utils.String("RServer"),
Configurations: gateway,
Configurations: configurations,
},
StorageProfile: &hdinsight.StorageProfile{
Storageaccounts: storageAccounts,
Expand Down Expand Up @@ -256,14 +265,20 @@ func resourceArmHDInsightRServerClusterRead(d *schema.ResourceData, meta interfa
return fmt.Errorf("Error retrieving HDInsight RServer Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

configuration, err := configurationsClient.Get(ctx, resourceGroup, name, "gateway")
// Each call to configurationsClient methods is HTTP request. Getting all settings in one operation
configurations, err := configurationsClient.List(ctx, resourceGroup, name)
if err != nil {
return fmt.Errorf("Error retrieving Gateway Configuration for HDInsight RServer Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
return fmt.Errorf("Error retrieving Configuration for HDInsight MLServices Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

rStudioConfig, err := configurationsClient.Get(ctx, resourceGroup, name, "rserver")
if err != nil {
return fmt.Errorf("Error retrieving RStudio Configuration for HDInsight RServer Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
gateway, exists := configurations.Configurations["gateway"]
if !exists {
return fmt.Errorf("Error retrieving gateway for HDInsight MLServices Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

rStudioConfig, exists := configurations.Configurations["rserver"]
if !exists {
return fmt.Errorf("Error retrieving RStudio Configuration for HDInsight MLServices Cluster %q (Resource Group %q): %+v", name, resourceGroup, err)
}

d.Set("name", name)
Expand All @@ -279,12 +294,14 @@ func resourceArmHDInsightRServerClusterRead(d *schema.ResourceData, meta interfa
d.Set("tls_min_version", props.MinSupportedTLSVersion)

if def := props.ClusterDefinition; def != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(configuration.Value)); err != nil {
if err := d.Set("gateway", azure.FlattenHDInsightsConfigurations(gateway)); err != nil {
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

flattenHDInsightsMetastores(d, configurations.Configurations)

var rStudio bool
if rStudioStr := rStudioConfig.Value["rstudio"]; rStudioStr != nil {
if rStudioStr := rStudioConfig["rstudio"]; rStudioStr != nil {
rStudioBool, err := strconv.ParseBool(*rStudioStr)
if err != nil {
return err
Expand Down
295 changes: 295 additions & 0 deletions azurerm/internal/services/hdinsight/tests/hdinsight_rserver_cluster_resource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,111 @@ func TestAccAzureRMHDInsightRServerCluster_tls(t *testing.T) {
})
}

func TestAccAzureRMHDInsightRServerCluster_allMetastores(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_hdinsight_rserver_cluster", "test")
resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acceptance.PreCheck(t) },
Providers: acceptance.SupportedProviders,
CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType),
Steps: []resource.TestStep{
{
Config: testAccAzureRMHDInsightRServerCluster_allMetastores(data),
Check: resource.ComposeTestCheckFunc(
testCheckAzureRMHDInsightClusterExists(data.ResourceName),
resource.TestCheckResourceAttrSet(data.ResourceName, "edge_ssh_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"),
),
},
data.ImportStep("roles.0.head_node.0.password",
"roles.0.head_node.0.vm_size",
"roles.0.worker_node.0.password",
"roles.0.worker_node.0.vm_size",
"roles.0.zookeeper_node.0.password",
"roles.0.zookeeper_node.0.vm_size",
"roles.0.edge_node.0.password",
"roles.0.edge_node.0.vm_size",
"storage_account",
"metastores.0.hive.0.password",
"metastores.0.oozie.0.password",
"metastores.0.ambari.0.password"),
},
})
}

func TestAccAzureRMHDInsightRServerCluster_hiveMetastore(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_hdinsight_rserver_cluster", "test")
resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acceptance.PreCheck(t) },
Providers: acceptance.SupportedProviders,
CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType),
Steps: []resource.TestStep{
{
Config: testAccAzureRMHDInsightRServerCluster_hiveMetastore(data),
Check: resource.ComposeTestCheckFunc(
testCheckAzureRMHDInsightClusterExists(data.ResourceName),
resource.TestCheckResourceAttrSet(data.ResourceName, "edge_ssh_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"),
),
},
},
})
}

func TestAccAzureRMHDInsightRServerCluster_updateMetastore(t *testing.T) {
data := acceptance.BuildTestData(t, "azurerm_hdinsight_rserver_cluster", "test")
resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acceptance.PreCheck(t) },
Providers: acceptance.SupportedProviders,
CheckDestroy: testCheckAzureRMHDInsightClusterDestroy(data.ResourceType),
Steps: []resource.TestStep{
{
Config: testAccAzureRMHDInsightRServerCluster_hiveMetastore(data),
Check: resource.ComposeTestCheckFunc(
testCheckAzureRMHDInsightClusterExists(data.ResourceName),
resource.TestCheckResourceAttrSet(data.ResourceName, "edge_ssh_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"),
),
},
data.ImportStep("roles.0.head_node.0.password",
"roles.0.head_node.0.vm_size",
"roles.0.worker_node.0.password",
"roles.0.worker_node.0.vm_size",
"roles.0.zookeeper_node.0.password",
"roles.0.zookeeper_node.0.vm_size",
"roles.0.edge_node.0.password",
"roles.0.edge_node.0.vm_size",
"storage_account",
"metastores.0.hive.0.password",
"metastores.0.oozie.0.password",
"metastores.0.ambari.0.password"),
{
Config: testAccAzureRMHDInsightRServerCluster_allMetastores(data),
Check: resource.ComposeTestCheckFunc(
testCheckAzureRMHDInsightClusterExists(data.ResourceName),
resource.TestCheckResourceAttrSet(data.ResourceName, "edge_ssh_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "https_endpoint"),
resource.TestCheckResourceAttrSet(data.ResourceName, "ssh_endpoint"),
),
},
data.ImportStep("roles.0.head_node.0.password",
"roles.0.head_node.0.vm_size",
"roles.0.worker_node.0.password",
"roles.0.worker_node.0.vm_size",
"roles.0.zookeeper_node.0.password",
"roles.0.zookeeper_node.0.vm_size",
"roles.0.edge_node.0.password",
"roles.0.edge_node.0.vm_size",
"storage_account",
"metastores.0.hive.0.password",
"metastores.0.oozie.0.password",
"metastores.0.ambari.0.password"),
},
})
}

func testAccAzureRMHDInsightRServerCluster_basic(data acceptance.TestData) string {
template := testAccAzureRMHDInsightRServerCluster_template(data)
return fmt.Sprintf(`
Expand Down Expand Up @@ -720,3 +825,193 @@ resource "azurerm_hdinsight_rserver_cluster" "test" {
}
`, template, data.RandomInteger)
}

func testAccAzureRMHDInsightRServerCluster_allMetastores(data acceptance.TestData) string {
template := testAccAzureRMHDInsightRServerCluster_template(data)
return fmt.Sprintf(`
%s
resource "azurerm_sql_server" "test" {
name = "acctestsql-%d"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
administrator_login = "sql_admin"
administrator_login_password = "TerrAform123!"
version = "12.0"
}
resource "azurerm_sql_database" "hive" {
name = "hive"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "oozie" {
name = "oozie"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "ambari" {
name = "ambari"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_firewall_rule" "AzureServices" {
name = "allow-azure-services"
resource_group_name = azurerm_resource_group.test.name
server_name = azurerm_sql_server.test.name
start_ip_address = "0.0.0.0"
end_ip_address = "0.0.0.0"
}
resource "azurerm_hdinsight_rserver_cluster" "test" {
name = "acctesthdi-%d"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
cluster_version = "3.6"
tier = "Standard"
rstudio = true
gateway {
enabled = true
username = "acctestusrgw"
password = "TerrAform123!"
}
storage_account {
storage_container_id = azurerm_storage_container.test.id
storage_account_key = azurerm_storage_account.test.primary_access_key
is_default = true
}
roles {
head_node {
vm_size = "Standard_D3_v2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
worker_node {
vm_size = "Standard_D4_V2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
target_instance_count = 2
}
zookeeper_node {
vm_size = "Standard_D3_v2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
edge_node {
vm_size = "Standard_D3_V2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
}
metastores {
hive {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.hive.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
oozie {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.oozie.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
ambari {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.ambari.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
}
}
`, template, data.RandomInteger, data.RandomInteger)
}

func testAccAzureRMHDInsightRServerCluster_hiveMetastore(data acceptance.TestData) string {
template := testAccAzureRMHDInsightRServerCluster_template(data)
return fmt.Sprintf(`
%s
resource "azurerm_sql_server" "test" {
name = "acctestsql-%d"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
administrator_login = "sql_admin"
administrator_login_password = "TerrAform123!"
version = "12.0"
}
resource "azurerm_sql_database" "hive" {
name = "hive"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_firewall_rule" "AzureServices" {
name = "allow-azure-services"
resource_group_name = azurerm_resource_group.test.name
server_name = azurerm_sql_server.test.name
start_ip_address = "0.0.0.0"
end_ip_address = "0.0.0.0"
}
resource "azurerm_hdinsight_rserver_cluster" "test" {
name = "acctesthdi-%d"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
cluster_version = "3.6"
tier = "Standard"
rstudio = true
gateway {
enabled = true
username = "acctestusrgw"
password = "TerrAform123!"
}
storage_account {
storage_container_id = azurerm_storage_container.test.id
storage_account_key = azurerm_storage_account.test.primary_access_key
is_default = true
}
roles {
head_node {
vm_size = "Standard_D3_v2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
worker_node {
vm_size = "Standard_D4_V2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
target_instance_count = 2
}
zookeeper_node {
vm_size = "Standard_D3_v2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
edge_node {
vm_size = "Standard_D3_V2"
username = "acctestusrvm"
password = "AccTestvdSC4daf986!"
}
}
metastores {
hive {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.hive.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
}
}
`, template, data.RandomInteger, data.RandomInteger)
}

0 comments on commit a4166b0

Please sign in to comment.