Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add num local ssds and boot disk types to preemptible worker config #2606

Closed
wants to merge 4 commits into from
Closed

Add num local ssds and boot disk types to preemptible worker config #2606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6a3977a
added num_local_ssds and boot_disk_type to preemptible_worker_config
cyrushanlon Dec 7, 2018
4b588dd
added tests
cyrushanlon Dec 7, 2018
d74e568
Fix test failure.
nat-henderson Dec 17, 2018
2435fc8
documentation for preemptible worker num_local_ssds and boot_disk_type
Dec 18, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions google/resource_dataproc_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,10 +215,12 @@ func resourceDataprocCluster() *schema.Resource {

Elem: &schema.Resource{
Schema: map[string]*schema.Schema{

// API does not honour this if set ...
// It simply ignores it completely
// "num_local_ssds": { ... }
"num_local_ssds": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
},

"boot_disk_size_gb": {
Type: schema.TypeInt,
Expand All @@ -227,6 +229,14 @@ func resourceDataprocCluster() *schema.Resource {
ForceNew: true,
ValidateFunc: validation.IntAtLeast(10),
},

"boot_disk_type": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"pd-standard", "pd-ssd", ""}, false),
Default: "pd-standard",
},
},
},
},
Expand Down Expand Up @@ -611,6 +621,12 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
if v, ok := dcfg["boot_disk_size_gb"]; ok {
icg.DiskConfig.BootDiskSizeGb = int64(v.(int))
}
if v, ok := dcfg["num_local_ssds"]; ok {
icg.DiskConfig.NumLocalSsds = int64(v.(int))
}
if v, ok := dcfg["boot_disk_type"]; ok {
icg.DiskConfig.BootDiskType = v.(string)
}
}
}
return icg
Expand Down Expand Up @@ -869,6 +885,8 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
data["instance_names"] = icg.InstanceNames
if icg.DiskConfig != nil {
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
}
}

Expand Down
4 changes: 4 additions & 0 deletions google/resource_dataproc_cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,8 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu

{"cluster_config.0.preemptible_worker_config.0.num_instances", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.NumInstances))},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb", "12", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskSizeGb))},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.NumLocalSsds))},
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskType},
{"cluster_config.0.preemptible_worker_config.0.instance_names.#", "1", strconv.Itoa(len(cluster.Config.SecondaryWorkerConfig.InstanceNames))},
}

Expand Down Expand Up @@ -851,7 +853,9 @@ resource "google_dataproc_cluster" "with_config_overrides" {
preemptible_worker_config {
num_instances = 1
disk_config {
boot_disk_type = "pd-ssd"
boot_disk_size_gb = 12
num_local_ssds = 1
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions website/docs/r/dataproc_cluster.html.markdown
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,9 @@ The `cluster_config.preemptible_worker_config` block supports:
preemptible_worker_config {
num_instances = 1
disk_config {
boot_disk_type = "pd-standard"
boot_disk_size_gb = 10
num_local_ssds = 1
}
}
}
Expand All @@ -357,11 +359,17 @@ will be set for you based on whatever was set for the `worker_config.machine_typ

* `disk_config` (Optional) Disk Config

* `boot_disk_type` - (Optional) The disk type of the primary disk attached to each preemptible worker node.
One of `"pd-ssd"` or `"pd-standard"`. Defaults to `"pd-standard"`.

* `boot_disk_size_gb` - (Optional, Computed) Size of the primary disk attached to each preemptible worker node, specified
in GB. The smallest allowed disk size is 10GB. GCP will default to a predetermined
computed value if not set (currently 500GB). Note: If SSDs are not
attached, it also contains the HDFS data blocks and Hadoop working directories.

* `num_local_ssds` - (Optional) The amount of local SSD disks that will be
attached to each preemptible worker node. Defaults to 0.

- - -

The `cluster_config.software_config` block supports:
Expand Down