Add Vertex AI Dataset

README.md

@@ -111,7 +111,7 @@ For Ansible and Inspec, wherever you have cloned those repositories.
 For example, to generate Terraform:
 
 ```bash
-bundle exec compiler -a -v "ga" -e terraform -o "$GOPATH/src/github.com/hashicorp/terraform-provider-google"
+bundle exec compiler -e terraform -o "$GOPATH/src/github.com/hashicorp/terraform-provider-google" -v "ga" -a 
 ```
 
 It's worth noting that Magic Modules will only generate new files when ran

mmv1/products/vertexai/api.yaml

@@ -0,0 +1,105 @@
+# Copyright 2019 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Api::Product
+name: VertexAI
+display_name: Vertex AI
+versions:
+  - !ruby/object:Api::Product::Version
+    name: ga
+    base_url: https://{{region}}-aiplatform.googleapis.com/v1/
+  - !ruby/object:Api::Product::Version
+    name: beta
+    base_url: https://{{region}}-aiplatform.googleapis.com/v1beta1/
+scopes:
+  - https://www.googleapis.com/auth/cloud-platform
+objects:
+# Vertex AI Datasets
+  - !ruby/object:Api::Resource
+    name: Dataset
+    base_url: projects/{{project}}/locations/{{region}}/datasets
+    self_link: '{{name}}'
+    update_verb: :PATCH
+    update_mask: true
+    references: !ruby/object:Api::Resource::ReferenceLinks
+      guides:
+        'Official Documentation':
+          'https://cloud.google.com/vertex-ai/docs'
+      api: 'https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.datasets'
+    async: !ruby/object:Api::OpAsync
+      operation: !ruby/object:Api::OpAsync::Operation
+        path: 'name'
+        base_url: '{{op_id}}'
+        wait_ms: 1000
+      result: !ruby/object:Api::OpAsync::Result
+        path: 'response'
+        resource_inside_response: true
+      status: !ruby/object:Api::OpAsync::Status
+        path: 'done'
+        complete: True
+        allowed:
+          - True
+          - False
+      error: !ruby/object:Api::OpAsync::Error
+        path: 'error'
+        message: 'message'
+    description: |-
+      A collection of DataItems and Annotations on them.
+    parameters:
+      - !ruby/object:Api::Type::String
+        name: region
+        description: The region of the dataset. eg us-central1
+        url_param_only: true
+        input: true
+    properties:
+      - !ruby/object:Api::Type::String
+        name: 'name'
+        description: The resource name of the Dataset. This value is set by Google.
+        output: true
+      - !ruby/object:Api::Type::String
+        name: 'displayName'
+        required: true
+        description: |
+          The user-defined name of the Dataset. The name can be up to 128 characters long and can be consist of any UTF-8 characters.
+      - !ruby/object:Api::Type::String
+        name: 'createTime'
+        output: true
+        description: |
+          The timestamp of when the workflow was created in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits.
+      - !ruby/object:Api::Type::String
+        name: 'updateTime'
+        output: true
+        description: |
+          The timestamp of when the workflow was last updated in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits.
+      - !ruby/object:Api::Type::KeyValuePairs
+        name: 'labels'
+        description: |
+          A set of key/value label pairs to assign to this Workflow.
+      - !ruby/object:Api::Type::NestedObject
+        name: 'encryptionSpec'
+        input: true
+        description: |
+          Customer-managed encryption key spec for a Dataset. If set, this Dataset and all sub-resources of this Dataset will be secured by this key.
+        properties:
+          - !ruby/object:Api::Type::String
+            name: 'kmsKeyName'
+            description: |
+              Required. The Cloud KMS resource identifier of the customer managed encryption key used to protect a resource. 
+              Has the form: projects/my-project/locations/my-region/keyRings/my-kr/cryptoKeys/my-key. The key needs to be in the same region as where the resource is created.
+            input: true
+      - !ruby/object:Api::Type::String
+        name: 'metadataSchemaUri'
+        required: true
+        input: true 
+        description: |
+          Points to a YAML file stored on Google Cloud Storage describing additional information about the Dataset. The schema is defined as an OpenAPI 3.0.2 Schema Object. The schema files that can be used here are found in gs://google-cloud-aiplatform/schema/dataset/metadata/.

mmv1/products/vertexai/terraform.yaml

@@ -0,0 +1,43 @@
+# Copyright 2021 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Provider::Terraform::Config
+overrides: !ruby/object:Overrides::ResourceOverrides
+  Dataset: !ruby/object:Overrides::Terraform::ResourceOverride
+    autogen_async: false
+    id_format: '{{name}}'
+    exclude_import: true
+    skip_sweeper: true
+    timeouts: !ruby/object:Api::Timeouts
+      insert_minutes: 6
+      update_minutes: 6
+      delete_minutes: 10
+    examples:
+      - !ruby/object:Provider::Terraform::Examples
+        name: "vertex_ai_dataset"
+        skip_import_test: true
+        primary_resource_id: "dataset"
+        vars:
+          display_name: "terraform"
+    properties:
+      labels: !ruby/object:Overrides::Terraform::PropertyOverride
+        default_from_api: true
+      region: !ruby/object:Overrides::Terraform::PropertyOverride
+        default_from_api: true
+
+# This is for copying files over
+files: !ruby/object:Provider::Config::Files
+  # These files have templating (ERB) code that will be run.
+  # This is usually to add licensing info, autogeneration notices, etc.
+  compile:
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

mmv1/templates/terraform/examples/vertex_ai_dataset.tf.erb

@@ -0,0 +1,5 @@
+resource "google_vertex_ai_dataset" "dataset" {
+  display_name          = "<%= ctx[:vars]['display_name'] %>"
+  metadata_schema_uri   = "gs://google-cloud-aiplatform/schema/dataset/metadata/image_1.0.0.yaml"
+  region                = "us-central1"
+}

mmv1/third_party/terraform/utils/self_link_helpers.go

@@ -169,3 +169,15 @@ func GetRegionFromRegionSelfLink(selfLink string) string {
 	}
 	return selfLink
 }
+
+// This function supports selflinks that have regions and locations in their paths
+func GetRegionFromRegionalSelfLink(selfLink string) string {
+	re := regexp.MustCompile("projects/[a-zA-Z0-9-]*/(?:locations|regions)/([a-zA-Z0-9-]*)")
+	switch {
+	case re.MatchString(selfLink):
+		if res := re.FindStringSubmatch(selfLink); len(res) == 2 && res[1] != "" {
+			return res[1]
+		}
+	}
+	return selfLink
+}

mmv1/third_party/terraform/utils/self_link_helpers_test.go

@@ -122,3 +122,15 @@ func TestGetRegionFromRegionSelfLink(t *testing.T) {
 		}
 	}
 }
+
+func TestGetRegionFromRegionalSelfLink(t *testing.T) {
+	cases := map[string]string{
+		"projects/foo/locations/europe-north1/datasets/bar/operations/foobar": "europe-north1",
+		"projects/REDACTED/regions/europe-north1/subnetworks/tf-test-net-xbwhsmlfm8": "europe-north1",
+	}
+	for input, expected := range cases {
+		if result := GetRegionFromRegionalSelfLink(input); result != expected {
+			t.Errorf("expected to get %q from %q, got %q", expected, input, result)
+		}
+	}
+}

mmv1/third_party/terraform/utils/vertex_ai_operation.go.erb

@@ -0,0 +1,69 @@
+<% autogen_exception -%>
+package google
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+type VertexAIOperationWaiter struct {
+	Config    *Config
+	UserAgent string
+	Project   string
+	CommonOperationWaiter
+}
+
+func (w *VertexAIOperationWaiter) QueryOp() (interface{}, error) {
+	if w == nil {
+		return nil, fmt.Errorf("Cannot query operation, it's unset or nil.")
+	}
+
+	region := GetRegionFromRegionalSelfLink(w.CommonOperationWaiter.Op.Name)
+
+	// Returns the proper get.
+<% if version == 'ga' -%>
+	url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1/%s", region, w.CommonOperationWaiter.Op.Name)
+<% else -%>
+	url := fmt.Sprintf("https://%s-aiplatform.googleapis.com/v1beta1/%s", region, w.CommonOperationWaiter.Op.Name)
+<% end -%>
+
+	return sendRequest(w.Config, "GET", w.Project, url, w.UserAgent, nil)
+}
+
+func createVertexAIWaiter(config *Config, op map[string]interface{}, project, activity, userAgent string) (*VertexAIOperationWaiter, error) {
+	w := &VertexAIOperationWaiter{
+		Config:    config,
+		UserAgent: userAgent,
+		Project:   project,
+	}
+	if err := w.CommonOperationWaiter.SetOp(op); err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// nolint: deadcode,unused
+func vertexAIOperationWaitTimeWithResponse(config *Config, op map[string]interface{}, response *map[string]interface{}, project, activity, userAgent string, timeout time.Duration) error {
+	w, err := createVertexAIWaiter(config, op, project, activity, userAgent)
+	if err != nil {
+		return err
+	}
+	if err := OperationWait(w, activity, timeout, config.PollInterval); err != nil {
+		return err
+	}
+	return json.Unmarshal([]byte(w.CommonOperationWaiter.Op.Response), response)
+}
+
+func vertexAIOperationWaitTime(config *Config, op map[string]interface{}, project, activity, userAgent string, timeout time.Duration) error {
+	if val, ok := op["name"]; !ok || val == "" {
+		// This was a synchronous call - there is no operation to wait for.
+		return nil
+	}
+	w, err := createVertexAIWaiter(config, op, project, activity, userAgent)
+	if err != nil {
+		// If w is nil, the op was synchronous.
+		return err
+	}
+	return OperationWait(w, activity, timeout, config.PollInterval)
+}