WIP: Add "internal IP only" support for Dataproc clusters (#837)

* Add internalIpOnly support for Dataproc clusters

* Add internal_ip_only to dataproc cluster docs

* Add default/basic dataproc internal ip test case

* Add test for dataproc internal_ip_only=true

* fixup cluster_config.gce_cluster_config to include .0.

* Remove redundant depends_on

* Add %s rnd to network and subnetwork

* Use variable for subnet CIDR and reference via source_ranges

* Add depends_on back to dataproc cluster test

* Fix cluster attribute refs (.0. again)

google/resource_dataproc_cluster.go

@@ -174,6 +174,13 @@ func resourceDataprocCluster() *schema.Resource {
 										},
 										Set: stringScopeHashcode,
 									},
+
+									"internal_ip_only": {
+										Type:     schema.TypeBool,
+										Optional: true,
+										ForceNew: true,
+										Default:  false,
+									},
 								},
 							},
 						},
@@ -479,6 +486,9 @@ func expandGceClusterConfig(cfg map[string]interface{}) *dataproc.GceClusterConf
 		}
 		conf.ServiceAccountScopes = scopes
 	}
+	if v, ok := cfg["internal_ip_only"]; ok {
+		conf.InternalIpOnly = v.(bool)
+	}
 	return conf
 }
 
@@ -720,9 +730,10 @@ func flattenInitializationActions(nia []*dataproc.NodeInitializationAction) ([]m
 func flattenGceClusterConfig(d *schema.ResourceData, gcc *dataproc.GceClusterConfig) []map[string]interface{} {
 
 	gceConfig := map[string]interface{}{
-		"tags":            gcc.Tags,
-		"service_account": gcc.ServiceAccount,
-		"zone":            extractLastResourceFromUri(gcc.ZoneUri),
+		"tags":             gcc.Tags,
+		"service_account":  gcc.ServiceAccount,
+		"zone":             extractLastResourceFromUri(gcc.ZoneUri),
+		"internal_ip_only": gcc.InternalIpOnly,
 	}
 
 	if gcc.NetworkUri != "" {

google/resource_dataproc_cluster_test.go

@@ -107,6 +107,9 @@ func TestAccDataprocCluster_basic(t *testing.T) {
 					// Default behaviour is for Dataproc to autogen or autodiscover a config bucket
 					resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.bucket"),
 
+					// Default behavior is for Dataproc to not use only internal IP addresses
+					resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "false"),
+
 					// Expect 1 master instances with computed values
 					resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.#", "1"),
 					resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.num_instances", "1"),
@@ -133,6 +136,29 @@ func TestAccDataprocCluster_basic(t *testing.T) {
 	})
 }
 
+func TestAccDataprocCluster_basicWithInternalIpOnlyTrue(t *testing.T) {
+	t.Parallel()
+
+	var cluster dataproc.Cluster
+	rnd := acctest.RandString(10)
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckDataprocClusterDestroy(false),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd),
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckDataprocClusterExists("google_dataproc_cluster.basic", &cluster),
+
+					// Testing behavior for Dataproc to use only internal IP addresses
+					resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "true"),
+				),
+			},
+		},
+	})
+}
+
 func TestAccDataprocCluster_basicWithAutogenDeleteTrue(t *testing.T) {
 	t.Parallel()
 
@@ -648,6 +674,72 @@ resource "google_dataproc_cluster" "basic" {
 `, rnd)
 }
 
+func testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd string) string {
+	return fmt.Sprintf(`
+variable subnetwork_cidr {
+	default = "10.0.0.0/16"
+}
+
+resource "google_compute_network" "dataproc_network" {
+	name = "dataproc-internalip-network-%s"
+	auto_create_subnetworks = false
+}
+
+#
+# Create a subnet with Private IP Access enabled to test
+# deploying a Dataproc cluster with Internal IP Only enabled.
+#
+resource "google_compute_subnetwork" "dataproc_subnetwork" {
+	name                     = "dataproc-internalip-subnetwork-%s"
+	ip_cidr_range            = "${var.subnetwork_cidr}"
+	network                  = "${google_compute_network.dataproc_network.self_link}"
+	region                   = "us-central1"
+	private_ip_google_access = true
+  }
+
+#
+# The default network within GCP already comes pre configured with
+# certain firewall rules open to allow internal communication. As we
+# are creating a new one here for this test, we need to additionally
+# open up similar rules to allow the nodes to talk to each other
+# internally as part of their configuration or this will just hang.
+#
+resource "google_compute_firewall" "dataproc_network_firewall" {
+	name = "dproc-cluster-test-allow-internal"
+	description = "Firewall rules for dataproc Terraform acceptance testing"
+	network = "${google_compute_network.dataproc_network.name}"
+
+	allow {
+		protocol = "icmp"
+	}
+
+	allow {
+		protocol = "tcp"
+		ports    = ["0-65535"]
+	}
+
+	allow {
+		protocol = "udp"
+		ports    = ["0-65535"]
+	}
+
+	source_ranges = ["${var.subnetwork_cidr}"]
+}
+resource "google_dataproc_cluster" "basic" {
+	name                  = "dproc-cluster-test-%s"
+	region                = "us-central1"
+	depends_on            = ["google_compute_firewall.dataproc_network_firewall"]
+	
+	cluster_config {
+		gce_cluster_config {
+			subnetwork       = "${google_compute_subnetwork.dataproc_subnetwork.name}"
+			internal_ip_only = true
+		}
+	}
+}
+`, rnd, rnd, rnd)
+}
+
 func testAccDataprocCluster_basicWithAutogenDeleteTrue(rnd string) string {
 	return fmt.Sprintf(`
 resource "google_dataproc_cluster" "basic" {

website/docs/r/dataproc_cluster.html.markdown

@@ -196,6 +196,11 @@ The **cluster_config.gce_cluster_config** block supports:
 * `tags` - (Optional) The list of instance tags applied to instances in the cluster.
    Tags are used to identify valid sources or targets for network firewalls.
 
+* `internal_ip_only` - (Optional) By default, clusters are not restricted to internal IP addresses, 
+   and will have ephemeral external IP addresses assigned to each instance. If set to true, all 
+   instances in the cluster will only have internal IP addresses. Note: Private Google Access 
+   (also known as `privateIpGoogleAccess`) must be enabled on the subnetwork that the cluster 
+   will be launched in.
 - - -
 
 The **cluster_config.master_config** block supports: