azurerm_hdinsight_*_cluster - Support for autoscale #8104

azurerm/internal/services/hdinsight/client/client.go

@@ -13,17 +13,21 @@ type Client struct {
 }
 
 func NewClient(o *common.ClientOptions) *Client {
-	ApplicationsClient := hdinsight.NewApplicationsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
-	o.ConfigureClient(&ApplicationsClient.Client, o.ResourceManagerAuthorizer)
+	// due to a bug in the HDInsight API we can't reuse client with the same x-ms-correlation-request-id for multiple updates
+	opts := *o
+	opts.DisableCorrelationRequestID = true
 
-	ClustersClient := hdinsight.NewClustersClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
-	o.ConfigureClient(&ClustersClient.Client, o.ResourceManagerAuthorizer)
+	ApplicationsClient := hdinsight.NewApplicationsClientWithBaseURI(opts.ResourceManagerEndpoint, opts.SubscriptionId)
+	opts.ConfigureClient(&ApplicationsClient.Client, opts.ResourceManagerAuthorizer)
 
-	ConfigurationsClient := hdinsight.NewConfigurationsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
-	o.ConfigureClient(&ConfigurationsClient.Client, o.ResourceManagerAuthorizer)
+	ClustersClient := hdinsight.NewClustersClientWithBaseURI(opts.ResourceManagerEndpoint, opts.SubscriptionId)
+	opts.ConfigureClient(&ClustersClient.Client, opts.ResourceManagerAuthorizer)
 
-	ExtensionsClient := hdinsight.NewExtensionsClientWithBaseURI(o.ResourceManagerEndpoint, o.SubscriptionId)
-	o.ConfigureClient(&ExtensionsClient.Client, o.ResourceManagerAuthorizer)
+	ConfigurationsClient := hdinsight.NewConfigurationsClientWithBaseURI(opts.ResourceManagerEndpoint, opts.SubscriptionId)
+	opts.ConfigureClient(&ConfigurationsClient.Client, opts.ResourceManagerAuthorizer)
+
+	ExtensionsClient := hdinsight.NewExtensionsClientWithBaseURI(opts.ResourceManagerEndpoint, opts.SubscriptionId)
+	opts.ConfigureClient(&ExtensionsClient.Client, opts.ResourceManagerAuthorizer)
 
 	return &Client{
 		ApplicationsClient:   &ApplicationsClient,

azurerm/internal/services/hdinsight/common_hdinsight.go

@@ -48,18 +48,37 @@ func hdinsightClusterUpdate(clusterKind string, readFunc schema.ReadFunc) schema
 			roles := rolesRaw[0].(map[string]interface{})
 			workerNodes := roles["worker_node"].([]interface{})
 			workerNode := workerNodes[0].(map[string]interface{})
-			targetInstanceCount := workerNode["target_instance_count"].(int)
-			params := hdinsight.ClusterResizeParameters{
-				TargetInstanceCount: utils.Int32(int32(targetInstanceCount)),
-			}
+			if d.HasChange("roles.0.worker_node.0.target_instance_count") {
+				targetInstanceCount := workerNode["target_instance_count"].(int)
+				params := hdinsight.ClusterResizeParameters{
+					TargetInstanceCount: utils.Int32(int32(targetInstanceCount)),
+				}
 
-			future, err := client.Resize(ctx, resourceGroup, name, params)
-			if err != nil {
-				return fmt.Errorf("Error resizing the HDInsight %q Cluster %q (Resource Group %q): %+v", clusterKind, name, resourceGroup, err)
+				future, err := client.Resize(ctx, resourceGroup, name, params)
+				if err != nil {
+					return fmt.Errorf("Error resizing the HDInsight %q Cluster %q (Resource Group %q): %+v", clusterKind, name, resourceGroup, err)
+				}
+
+				if err = future.WaitForCompletionRef(ctx, client.Client); err != nil {
+					return fmt.Errorf("Error waiting for the HDInsight %q Cluster %q (Resource Group %q) to finish resizing: %+v", clusterKind, name, resourceGroup, err)
+				}
 			}
 
-			if err = future.WaitForCompletionRef(ctx, client.Client); err != nil {
-				return fmt.Errorf("Error waiting for the HDInsight %q Cluster %q (Resource Group %q) to finish resizing: %+v", clusterKind, name, resourceGroup, err)
+			if d.HasChange("roles.0.worker_node.0.autoscale") {
+				autoscale := ExpandHDInsightNodeAutoScaleDefinition(workerNode["autoscale"].([]interface{}))
+				params := hdinsight.AutoscaleConfigurationUpdateParameter{
+					Autoscale: autoscale,
+				}
+
+				future, err := client.UpdateAutoScaleConfiguration(ctx, resourceGroup, name, params)
+
+				if err != nil {
+					return fmt.Errorf("Error changing autoscale of the HDInsight %q Cluster %q (Resource Group %q): %+v", clusterKind, name, resourceGroup, err)
+				}
+
+				if err = future.WaitForCompletionRef(ctx, client.Client); err != nil {
+					return fmt.Errorf("Error waiting for changing autoscale of the HDInsight %q Cluster %q (Resource Group %q) to finish resizing: %+v", clusterKind, name, resourceGroup, err)
+				}
 			}
 		}
 

azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource.go

@@ -37,6 +37,8 @@ var hdInsightHadoopClusterWorkerNodeDefinition = HDInsightNodeDefinition{
 	CanSpecifyInstanceCount: true,
 	MinInstanceCount:        1,
 	CanSpecifyDisks:         false,
+	CanAutoScaleByCapacity:  true,
+	CanAutoScaleOnSchedule:  true,
 }
 
 var hdInsightHadoopClusterZookeeperNodeDefinition = HDInsightNodeDefinition{

azurerm/internal/services/hdinsight/hdinsight_hadoop_cluster_resource_test.go

@@ -541,6 +541,58 @@ func TestAccHDInsightHadoopCluster_updateMonitor(t *testing.T) {
 	})
 }
 
+func TestAccAzureRMHDInsightHadoopCluster_autoscale(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_hdinsight_hadoop_cluster", "test")
+	r := HDInsightHadoopClusterResource{}
+	data.ResourceTest(t, r, []resource.TestStep{
+		{
+			Config: r.autoscale_schedule(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("https_endpoint").Exists(),
+				check.That(data.ResourceName).Key("ssh_endpoint").Exists(),
+			),
+		},
+		data.ImportStep("roles.0.head_node.0.password",
+			"roles.0.head_node.0.vm_size",
+			"roles.0.worker_node.0.password",
+			"roles.0.worker_node.0.vm_size",
+			"roles.0.zookeeper_node.0.password",
+			"roles.0.zookeeper_node.0.vm_size",
+			"storage_account"),
+		{
+			Config: r.autoscale_capacity(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("https_endpoint").Exists(),
+				check.That(data.ResourceName).Key("ssh_endpoint").Exists(),
+			),
+		},
+		data.ImportStep("roles.0.head_node.0.password",
+			"roles.0.head_node.0.vm_size",
+			"roles.0.worker_node.0.password",
+			"roles.0.worker_node.0.vm_size",
+			"roles.0.zookeeper_node.0.password",
+			"roles.0.zookeeper_node.0.vm_size",
+			"storage_account"),
+		{
+			Config: r.basic(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("https_endpoint").Exists(),
+				check.That(data.ResourceName).Key("ssh_endpoint").Exists(),
+			),
+		},
+		data.ImportStep("roles.0.head_node.0.password",
+			"roles.0.head_node.0.vm_size",
+			"roles.0.worker_node.0.password",
+			"roles.0.worker_node.0.vm_size",
+			"roles.0.zookeeper_node.0.password",
+			"roles.0.zookeeper_node.0.vm_size",
+			"storage_account"),
+	})
+}
+
 func (t HDInsightHadoopClusterResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
 	id, err := parse.ClusterID(state.ID)
 	if err != nil {
@@ -1862,3 +1914,110 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" {
 }
 `, r.template(data), data.RandomInteger)
 }
+
+func (r HDInsightHadoopClusterResource) autoscale_capacity(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+%s
+resource "azurerm_hdinsight_hadoop_cluster" "test" {
+  name                = "acctesthdi-%d"
+  resource_group_name = azurerm_resource_group.test.name
+  location            = azurerm_resource_group.test.location
+  cluster_version     = "4.0"
+  tier                = "Standard"
+  component_version {
+    hadoop = "3.1"
+  }
+  gateway {
+    username = "acctestusrgw"
+    password = "TerrAform123!"
+  }
+  storage_account {
+    storage_container_id = azurerm_storage_container.test.id
+    storage_account_key  = azurerm_storage_account.test.primary_access_key
+    is_default           = true
+  }
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+    worker_node {
+      vm_size               = "Standard_D4_V2"
+      username              = "acctestusrvm"
+      password              = "AccTestvdSC4daf986!"
+      target_instance_count = 2
+      autoscale {
+        capacity {
+          min_instance_count = 2
+          max_instance_count = 3
+        }
+      }
+    }
+    zookeeper_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+  }
+}
+`, r.template(data), data.RandomInteger)
+}
+
+func (r HDInsightHadoopClusterResource) autoscale_schedule(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+%s
+resource "azurerm_hdinsight_hadoop_cluster" "test" {
+  name                = "acctesthdi-%d"
+  resource_group_name = azurerm_resource_group.test.name
+  location            = azurerm_resource_group.test.location
+  cluster_version     = "4.0"
+  tier                = "Standard"
+  component_version {
+    hadoop = "3.1"
+  }
+  gateway {
+    username = "acctestusrgw"
+    password = "TerrAform123!"
+  }
+  storage_account {
+    storage_container_id = azurerm_storage_container.test.id
+    storage_account_key  = azurerm_storage_account.test.primary_access_key
+    is_default           = true
+  }
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+    worker_node {
+      vm_size               = "Standard_D4_V2"
+      username              = "acctestusrvm"
+      password              = "AccTestvdSC4daf986!"
+      target_instance_count = 2
+      autoscale {
+        recurrence {
+          timezone = "Pacific Standard Time"
+          schedule {
+            days                  = ["Monday"]
+            time                  = "10:00"
+            target_instance_count = 5
+          }
+          schedule {
+            days                  = ["Saturday", "Sunday"]
+            time                  = "10:00"
+            target_instance_count = 3
+          }
+        }
+      }
+    }
+    zookeeper_node {
+      vm_size  = "Standard_D3_v2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+  }
+}
+`, r.template(data), data.RandomInteger)
+}

azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource.go

@@ -32,6 +32,7 @@ var hdInsightHBaseClusterWorkerNodeDefinition = HDInsightNodeDefinition{
 	CanSpecifyInstanceCount: true,
 	MinInstanceCount:        1,
 	CanSpecifyDisks:         false,
+	CanAutoScaleOnSchedule:  true,
 }
 
 var hdInsightHBaseClusterZookeeperNodeDefinition = HDInsightNodeDefinition{

azurerm/internal/services/hdinsight/hdinsight_hbase_cluster_resource_test.go

@@ -382,6 +382,43 @@ func TestAccHDInsightHBaseCluster_updateMonitor(t *testing.T) {
 	})
 }
 
+func TestAccAzureRMHDInsightHBaseCluster_autoscale(t *testing.T) {
+	data := acceptance.BuildTestData(t, "azurerm_hdinsight_hbase_cluster", "test")
+	r := HDInsightHBaseClusterResource{}
+	data.ResourceTest(t, r, []resource.TestStep{
+		{
+			Config: r.autoscale_schedule(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("https_endpoint").Exists(),
+				check.That(data.ResourceName).Key("ssh_endpoint").Exists(),
+			),
+		},
+		data.ImportStep("roles.0.head_node.0.password",
+			"roles.0.head_node.0.vm_size",
+			"roles.0.worker_node.0.password",
+			"roles.0.worker_node.0.vm_size",
+			"roles.0.zookeeper_node.0.password",
+			"roles.0.zookeeper_node.0.vm_size",
+			"storage_account"),
+		{
+			Config: r.basic(data),
+			Check: resource.ComposeTestCheckFunc(
+				check.That(data.ResourceName).ExistsInAzure(r),
+				check.That(data.ResourceName).Key("https_endpoint").Exists(),
+				check.That(data.ResourceName).Key("ssh_endpoint").Exists(),
+			),
+		},
+		data.ImportStep("roles.0.head_node.0.password",
+			"roles.0.head_node.0.vm_size",
+			"roles.0.worker_node.0.password",
+			"roles.0.worker_node.0.vm_size",
+			"roles.0.zookeeper_node.0.password",
+			"roles.0.zookeeper_node.0.vm_size",
+			"storage_account"),
+	})
+}
+
 func (t HDInsightHBaseClusterResource) Exists(ctx context.Context, clients *clients.Client, state *terraform.InstanceState) (*bool, error) {
 	id, err := parse.ClusterID(state.ID)
 	if err != nil {
@@ -1200,3 +1237,69 @@ resource "azurerm_hdinsight_hbase_cluster" "test" {
 }
 `, r.template(data), data.RandomString, data.RandomInteger, data.RandomInteger)
 }
+
+func (r HDInsightHBaseClusterResource) autoscale_schedule(data acceptance.TestData) string {
+	return fmt.Sprintf(`
+%s
+
+resource "azurerm_hdinsight_hbase_cluster" "test" {
+  name                = "acctesthdi-%d"
+  resource_group_name = azurerm_resource_group.test.name
+  location            = azurerm_resource_group.test.location
+  cluster_version     = "4.0"
+  tier                = "Standard"
+
+  component_version {
+    hbase = "2.1"
+  }
+
+  gateway {
+    enabled  = true
+    username = "acctestusrgw"
+    password = "TerrAform123!"
+  }
+
+  storage_account {
+    storage_container_id = azurerm_storage_container.test.id
+    storage_account_key  = azurerm_storage_account.test.primary_access_key
+    is_default           = true
+  }
+
+  roles {
+    head_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+
+    worker_node {
+      vm_size               = "Standard_D3_V2"
+      username              = "acctestusrvm"
+      password              = "AccTestvdSC4daf986!"
+      target_instance_count = 2
+      autoscale {
+        recurrence {
+          timezone = "Pacific Standard Time"
+          schedule {
+            days                  = ["Monday"]
+            time                  = "10:00"
+            target_instance_count = 5
+          }
+          schedule {
+            days                  = ["Saturday", "Sunday"]
+            time                  = "10:00"
+            target_instance_count = 3
+          }
+        }
+      }
+    }
+
+    zookeeper_node {
+      vm_size  = "Standard_D3_V2"
+      username = "acctestusrvm"
+      password = "AccTestvdSC4daf986!"
+    }
+  }
+}
+`, r.template(data), data.RandomInteger)
+}

azurerm/internal/services/hdinsight/hdinsight_interactive_query_cluster_resource.go

@@ -32,6 +32,8 @@ var hdInsightInteractiveQueryClusterWorkerNodeDefinition = HDInsightNodeDefiniti
 	CanSpecifyInstanceCount: true,
 	MinInstanceCount:        1,
 	CanSpecifyDisks:         false,
+	CanAutoScaleByCapacity:  true,
+	CanAutoScaleOnSchedule:  true,
 }
 
 var hdInsightInteractiveQueryClusterZookeeperNodeDefinition = HDInsightNodeDefinition{