Dataproc workflow template (#4693)

* Update eventarc trigger yaml, markdown to allow setting transport

* Update test, add handwritten utils

* Update handwritten

* Updating eventarc test

* Updating trigger

* Adding dataproc workflow

* Updating expanders

* Updates to prop, resource to handle refs

* Add support for workflow in GA

* fix merge issues

* Add template markdown

mmv1/third_party/terraform/tests/resource_dataproc_workflow_template_test.go.erb

@@ -0,0 +1,127 @@
+<% autogen_exception -%>
+package google
+
+import (
+  "testing"
+
+  "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+)
+
+func TestAccDataprocWorkflowTemplate_basic(t *testing.T) {
+  // DCL currently fails due to transport modification
+  skipIfVcr(t)
+  t.Parallel()
+
+  context := map[string]interface{}{
+    "random_suffix": randString(t, 10),
+    "project": getTestProjectFromEnv(),
+  }
+
+  vcrTest(t, resource.TestCase{
+    PreCheck:          func() { testAccPreCheck(t) },
+    Providers:         testAccProviders,
+    CheckDestroy:      funcAccTestDataprocWorkflowTemplateCheckDestroy(t),
+    ExternalProviders: map[string]resource.ExternalProvider{
+      "random": {},
+    },
+    Steps: []resource.TestStep{
+      {
+        Config: testAccDataprocWorkflowTemplate_basic(context),
+      },
+      {
+        ImportState:             true,
+        ImportStateVerify:       true,
+        ResourceName:            "google_dataproc_workflow_template.template",
+      },
+    },
+  })
+}
+
+func testAccDataprocWorkflowTemplate_basic(context map[string]interface{}) string {
+  return Nprintf(`
+resource "google_dataproc_workflow_template" "template" {
+  name = "template%{random_suffix}"
+  location = "us-central1"
+  placement {
+    managed_cluster {
+      cluster_name = "my-cluster"
+      config {
+        gce_cluster_config {
+          zone = "us-central1-a"
+          tags = ["foo", "bar"]
+        }
+        master_config {
+          num_instances = 1
+          machine_type = "n1-standard-1"
+          disk_config {
+            boot_disk_type = "pd-ssd"
+            boot_disk_size_gb = 15
+          }
+        }
+        worker_config {
+          num_instances = 3
+          machine_type = "n1-standard-2"
+          disk_config {
+            boot_disk_size_gb = 10
+            num_local_ssds = 2
+          }
+        }
+
+        secondary_worker_config {
+          num_instances = 2
+        }
+        software_config {
+          image_version = "1.3.7-deb9"
+        }
+      }
+    }
+  }
+  jobs {
+    step_id = "someJob"
+    spark_job {
+      main_class = "SomeClass"
+    }
+  }
+  jobs {
+    step_id = "otherJob"
+    prerequisite_step_ids = ["someJob"]
+    presto_job {
+      query_file_uri = "someuri"
+    }
+  }
+}
+`, context)
+}
+
+func funcAccTestDataprocWorkflowTemplateCheckDestroy(t *testing.T) func(s *terraform.State) error {
+  return func(s *terraform.State) error {
+    for name, rs := range s.RootModule().Resources {
+      if rs.Type != "google_dataproc_workflow_template" {
+        continue
+      }
+      if strings.HasPrefix(name, "data.") {
+        continue
+      }
+
+      config := googleProviderConfig(t)
+
+      url, err := replaceVarsForTest(config, rs, "{{DataprocBasePath}}projects/{{project}}/locations/{{location}}/workflowTemplates/{{name}}")
+      if err != nil {
+        return err
+      }
+
+      billingProject := ""
+
+      if config.BillingProject != "" {
+        billingProject = config.BillingProject
+      }
+
+      _, err = sendRequest(config, "GET", billingProject, url, config.userAgent, nil)
+      if err == nil {
+        return fmt.Errorf("DataprocWorkflowTemplate still exists at %s", url)
+      }
+    }
+
+    return nil
+  }
+}

mmv1/third_party/terraform/utils/config.go.erb

@@ -64,6 +64,7 @@ import (
 	"google.golang.org/api/storagetransfer/v1"
 	"google.golang.org/api/transport"
 	dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"
+	dataprocDcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/dataproc<% unless version == 'ga' -%>/beta<% end -%>"
 	eventarcDcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/eventarc<% unless version == 'ga' -%>/beta<% end -%>"
 )
 
@@ -121,6 +122,7 @@ type Config struct {
 
 	// start DCL clients
 	dclConfig *dcl.Config
+	clientDataprocDCL *dataprocDcl.Client
 	clientEventarcDCL *eventarcDcl.Client
 }
 
@@ -186,6 +188,7 @@ func (c *Config) LoadAndValidate(ctx context.Context) error {
 	// Start DCL client instantiation
 	// TODO(slevenick): handle user agents
 	c.dclConfig = dcl.NewConfig(dcl.WithHTTPClient(client), dcl.WithUserAgent(c.userAgent), dcl.WithLogger(dclLogger{}))
+	c.clientDataprocDCL = dataprocDcl.NewClient(c.dclConfig)
 	c.clientEventarcDCL = eventarcDcl.NewClient(c.dclConfig)
 
 	return nil

mmv1/third_party/terraform/utils/provider.go.erb

@@ -365,6 +365,7 @@ end     # products.each do
 				<% end -%>
 				"google_dataproc_cluster":                      resourceDataprocCluster(),
 				"google_dataproc_job":                          resourceDataprocJob(),
+				"google_dataproc_workflow_template":            resourceDataprocWorkflowTemplate(),
 				"google_endpoints_service":                     resourceEndpointsService(),
 				"google_eventarc_trigger":                      resourceEventarcTrigger(),
 				"google_folder":                                resourceGoogleFolder(),

mmv1/third_party/terraform/utils/utils.go

@@ -482,14 +482,6 @@ func SnakeToPascalCase(s string) string {
 	return strings.Join(split, "")
 }
 
-func checkStringMap(v interface{}) map[string]string {
-	m, ok := v.(map[string]string)
-	if ok {
-		return m
-	}
-	return convertStringMap(v.(map[string]interface{}))
-}
-
 func multiEnvSearch(ks []string) string {
 	for _, k := range ks {
 		if v := os.Getenv(k); v != "" {
@@ -498,3 +490,11 @@ func multiEnvSearch(ks []string) string {
 	}
 	return ""
 }
+
+func checkStringMap(v interface{}) map[string]string {
+	m, ok := v.(map[string]string)
+	if ok {
+		return m
+	}
+	return convertStringMap(v.(map[string]interface{}))
+}