Skip to content

Commit

Permalink
add support for instance_flexibility_policy in google_dataproc_cluster (
Browse files Browse the repository at this point in the history 
GoogleCloudPlatform#9269)

Co-authored-by: Cameron Thornton <[email protected]>
Co-authored-by: sunchengxuanivy <[email protected]>
  • Loading branch information
@sunchengxuanivy @c2thorn @sunivycx
3 people authored Oct 31, 2023
1 parent 104bd70 commit f173603
Show file tree
Hide file tree
Showing 4 changed files with 286 additions and 1 deletion.
136 changes: 136 additions & 0 deletions mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1139,6 +1139,74 @@ func ResourceDataprocCluster() *schema.Resource {
Elem: &schema.Schema{Type: schema.TypeString},
Description: `List of preemptible instance names which have been assigned to the cluster.`,
},
"instance_flexibility_policy": {
Type: schema.TypeList,
Optional: true,
Computed: true,
Description: `Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.`,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.num_instances",
"cluster_config.0.preemptible_worker_config.0.preemptibility",
"cluster_config.0.preemptible_worker_config.0.disk_config",
"cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy",
},
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"instance_selection_list": {
Type: schema.TypeList,
Computed: true,
Optional: true,
ForceNew: true,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list",
},
Description: `List of instance selection options that the group will use when creating new VMs.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"machine_types": {
Type: schema.TypeList,
Computed: true,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
Description: `Full machine-type names, e.g. "n1-standard-16".`,
},
"rank": {
Type: schema.TypeInt,
Computed: true,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeInt},
Description: `Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.`,
},
},
},
},
"instance_selection_results": {
Type: schema.TypeList,
Computed: true,
Description: `A list of instance selection results in the group.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"machine_type": {
Type: schema.TypeString,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeString},
Description: `Full machine-type names, e.g. "n1-standard-16".`,
},
"vm_count": {
Type: schema.TypeInt,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeInt},
Description: `Number of VM provisioned with the machine_type.`,
},
},
},
},
},
},
},
},
},
},
Expand Down Expand Up @@ -2095,12 +2163,46 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
}
}
}

if ifpc, ok := cfg["instance_flexibility_policy"]; ok {
ifps := ifpc.([]interface{})
if len(ifps) > 0 {
flexibilityPolicy := ifps[0].(map[string]interface{})
icg.InstanceFlexibilityPolicy = &dataproc.InstanceFlexibilityPolicy{}
if v, ok := flexibilityPolicy["instance_selection_list"]; ok {
icg.InstanceFlexibilityPolicy.InstanceSelectionList = expandInstanceSelectionList(v)
}
}

}
if p, ok := cfg["preemptibility"]; ok {
icg.Preemptibility = p.(string)
}
return icg
}

func expandInstanceSelectionList(v interface{}) []*dataproc.InstanceSelection {
instanceSelectionList := v.([]interface{})

instanceSelections := []*dataproc.InstanceSelection{}
for _, v1 := range instanceSelectionList {
instanceSelectionItem := v1.(map[string]interface{})
machineTypes := []string{}
for _, machineType := range instanceSelectionItem["machine_types"].([]interface{}) {
machineTypes = append(machineTypes, machineType.(string))
}
instanceSelection := &dataproc.InstanceSelection{
MachineTypes: machineTypes,
}
if x, ok := instanceSelectionItem["rank"]; ok {
instanceSelection.Rank = int64(x.(int))
}
instanceSelections = append(instanceSelections, instanceSelection)
}

return instanceSelections
}

func expandMasterInstanceGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGroupConfig {
icg := &dataproc.InstanceGroupConfig{}

Expand Down Expand Up @@ -2752,6 +2854,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
}

disk := map[string]interface{}{}
instanceFlexibilityPolicy := map[string]interface{}{}
data := map[string]interface{}{}

if icg != nil {
Expand All @@ -2763,12 +2866,45 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
}
if icg.InstanceFlexibilityPolicy != nil {
instanceFlexibilityPolicy["instance_selection_list"] = flattenInstanceSelectionList(icg.InstanceFlexibilityPolicy.InstanceSelectionList)
instanceFlexibilityPolicy["instance_selection_results"] = flattenInstanceSelectionResults(icg.InstanceFlexibilityPolicy.InstanceSelectionResults)
}
}

data["disk_config"] = []map[string]interface{}{disk}
data["instance_flexibility_policy"] = []map[string]interface{}{instanceFlexibilityPolicy}
return []map[string]interface{}{data}
}

func flattenInstanceSelectionList(is []*dataproc.InstanceSelection) []map[string]interface{} {

instanceSelections := []map[string]interface{}{}
for _, v := range is {
instanceSelection := map[string]interface{}{}
if len(v.MachineTypes) > 0 {
instanceSelection["machine_types"] = v.MachineTypes
}
instanceSelection["rank"] = v.Rank
instanceSelections = append(instanceSelections, instanceSelection)
}
return instanceSelections

}

func flattenInstanceSelectionResults(isr []*dataproc.InstanceSelectionResult) []map[string]interface{} {

instanceSelectionResults := []map[string]interface{}{}
for _, v := range isr {
instanceSelection := map[string]interface{}{}
instanceSelection["machine_type"] = v.MachineType
instanceSelection["vm_count"] = v.VmCount
instanceSelectionResults = append(instanceSelectionResults, instanceSelection)
}
return instanceSelectionResults

}

func flattenMasterInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.InstanceGroupConfig) []map[string]interface{} {
disk := map[string]interface{}{}
data := map[string]interface{}{}
Expand Down
68 changes: 68 additions & 0 deletions mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -961,6 +961,74 @@ func resourceDataprocClusterResourceV0() *schema.Resource {
Elem: &schema.Schema{Type: schema.TypeString},
Description: `List of preemptible instance names which have been assigned to the cluster.`,
},
"instance_flexibility_policy": {
Type: schema.TypeList,
Optional: true,
Computed: true,
Description: `Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.`,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.num_instances",
"cluster_config.0.preemptible_worker_config.0.preemptibility",
"cluster_config.0.preemptible_worker_config.0.disk_config",
"cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy",
},
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"instance_selection_list": {
Type: schema.TypeList,
Computed: true,
Optional: true,
ForceNew: true,
AtLeastOneOf: []string{
"cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list",
},
Description: `List of instance selection options that the group will use when creating new VMs.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"machine_types": {
Type: schema.TypeList,
Computed: true,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
Description: `Full machine-type names, e.g. "n1-standard-16".`,
},
"rank": {
Type: schema.TypeInt,
Computed: true,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeInt},
Description: `Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.`,
},
},
},
},
"instance_selection_results": {
Type: schema.TypeList,
Computed: true,
Description: `A list of instance selection results in the group.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"machine_type": {
Type: schema.TypeString,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeString},
Description: `Full machine-type names, e.g. "n1-standard-16".`,
},
"vm_count": {
Type: schema.TypeInt,
Computed: true,
Elem: &schema.Schema{Type: schema.TypeInt},
Description: `Number of VM provisioned with the machine_type.`,
},
},
},
},
},
},
},
},
},
},
Expand Down
64 changes: 64 additions & 0 deletions mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,6 +491,29 @@ func TestAccDataprocCluster_spotSecondary(t *testing.T) {
})
}

func TestAccDataprocCluster_spotWithInstanceFlexibilityPolicy(t *testing.T) {
t.Parallel()

rnd := acctest.RandString(t, 10)
var cluster dataproc.Cluster
acctest.VcrTest(t, resource.TestCase{
PreCheck: func() { acctest.AccTestPreCheck(t) },
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
CheckDestroy: testAccCheckDataprocClusterDestroy(t),
Steps: []resource.TestStep{
{
Config: testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd),
Check: resource.ComposeTestCheckFunc(
testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.spot_with_instance_flexibility_policy", &cluster),
resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.preemptibility", "SPOT"),
resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "n2d-standard-2"),
resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "3"),
),
},
},
})
}

func TestAccDataprocCluster_withStagingBucket(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -1817,6 +1840,47 @@ resource "google_dataproc_cluster" "spot_secondary" {
`, rnd, subnetworkName)
}

func testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd string) string {
return fmt.Sprintf(`
resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" {
name = "tf-test-dproc-%s"
region = "us-central1"

cluster_config {
master_config {
num_instances = "1"
machine_type = "e2-medium"
disk_config {
boot_disk_size_gb = 35
}
}

worker_config {
num_instances = "2"
machine_type = "e2-medium"
disk_config {
boot_disk_size_gb = 35
}
}

preemptible_worker_config {
num_instances = "3"
preemptibility = "SPOT"
disk_config {
boot_disk_size_gb = 35
}
instance_flexibility_policy {
instance_selection_list {
machine_types = ["n2d-standard-2"]
rank = 3
}
}
}
}
}
`, rnd)
}

func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string {
return fmt.Sprintf(`
resource "google_storage_bucket" "bucket" {
Expand Down
19 changes: 18 additions & 1 deletion mmv1/third_party/terraform/website/docs/r/dataproc_cluster.html.markdown
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ Manages a Cloud Dataproc cluster resource within GCP.


!> **Warning:** Due to limitations of the API, all arguments except
`labels`,`cluster_config.worker_config.num_instances` and `cluster_config.preemptible_worker_config.num_instances` are non-updatable. Changing others will cause recreation of the
`labels`,`cluster_config.worker_config.num_instances` and `cluster_config.preemptible_worker_config.num_instances` are non-updatable. Changing `cluster_config.worker_config.min_num_instances` will be ignored. Changing others will cause recreation of the
whole cluster!

## Example Usage - Basic
Expand Down Expand Up @@ -608,6 +608,16 @@ cluster_config {
boot_disk_size_gb = 30
num_local_ssds = 1
}
instance_flexibility_policy {
instance_selection_list {
machine_types = ["n2-standard-2","n1-standard-2"]
rank = 1
}
instance_selection_list {
machine_types = ["n2d-standard-2"]
rank = 3
}
}
}
}
```
Expand Down Expand Up @@ -638,6 +648,13 @@ will be set for you based on whatever was set for the `worker_config.machine_typ
* `num_local_ssds` - (Optional) The amount of local SSD disks that will be
attached to each preemptible worker node. Defaults to 0.

* `instance_flexibility_policy` (Optional) Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.

* `instance_selection_list` - (Optional) List of instance selection options that the group will use when creating new VMs.
* `machine_types` - (Optional) Full machine-type names, e.g. `"n1-standard-16"`.

* `rank` - (Optional) Preference of this instance selection. A lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.

- - -

<a name="nested_software_config"></a>The `cluster_config.software_config` block supports:
Expand Down

0 comments on commit f173603

Please sign in to comment.