Add Cloud TPU to Terraform (#1360)

Merged PR #1360.

products/tpu/api.yaml

@@ -0,0 +1,155 @@
+# Copyright 2018 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Api::Product
+name: Tpu
+display_name: Google Cloud TPU
+versions:
+  - !ruby/object:Api::Product::Version
+    name: ga
+    base_url: https://tpu.googleapis.com/v1/
+    default: true
+scopes:
+  - https://www.googleapis.com/auth/cloud-platform
+objects:
+  - !ruby/object:Api::Resource
+    name: 'Node'
+    input: true
+    base_url: projects/{{project}}/locations/{{zone}}/nodes
+    create_url: projects/{{project}}/locations/{{zone}}/nodes?nodeId={{name}}
+    self_link: projects/{{project}}/locations/{{zone}}/nodes/{{name}}
+    description: |
+      A Cloud TPU instance.
+    references: !ruby/object:Api::Resource::ReferenceLinks
+      guides:
+        'Official Documentation':
+          'https://cloud.google.com/tpu/docs/'
+      api: 'https://cloud.google.com/tpu/docs/reference/rest/'
+    async: !ruby/object:Api::Async
+      operation: !ruby/object:Api::Async::Operation
+        path: 'name'
+        base_url: '{{op_id}}'
+        wait_ms: 1000
+        timeouts: !ruby/object:Api::Timeouts
+          insert_sec: 900
+          update_sec: 900
+          delete_sec: 900
+      result: !ruby/object:Api::Async::Result
+        path: 'response'
+        resource_inside_response: true
+      status: !ruby/object:Api::Async::Status
+        path: 'done'
+        complete: true
+        allowed:
+          - true
+          - false
+      error: !ruby/object:Api::Async::Error
+        path: 'error'
+        message: 'message'
+    parameters:
+      - !ruby/object:Api::Type::String # TODO: resourceref?
+        name: 'zone'
+        description: |
+          The GCP location for the TPU.
+        required: true
+        input: true
+        url_param_only: true
+    properties:
+      - !ruby/object:Api::Type::String
+        name: name
+        required: true
+        input: true
+        description: |
+          The immutable name of the TPU.
+      - !ruby/object:Api::Type::String
+        name: 'description'
+        input: true
+        description: |
+          The user-supplied description of the TPU. Maximum of 512 characters.
+      - !ruby/object:Api::Type::String
+        name: 'acceleratorType'
+        required: true
+        input: true
+        description: |
+          The type of hardware accelerators associated with this node.
+      - !ruby/object:Api::Type::String
+        name: 'tensorflowVersion'
+        required: true
+        update_url: 'projects/{{project}}/locations/{{zone}}/nodes/{{name}}:reimage'
+        update_verb: :POST
+        description: |
+          The version of Tensorflow running in the Node.
+      - !ruby/object:Api::Type::String
+        name: 'network'
+        input: true
+        description: |
+          The name of a network to peer the TPU node to. It must be a
+          preexisting Compute Engine network inside of the project on which
+          this API has been activated. If none is provided, "default" will be
+          used.
+      - !ruby/object:Api::Type::String
+        name: 'cidrBlock'
+        required: true
+        input: true
+        description: |
+          The CIDR block that the TPU node will use when selecting an IP
+          address. This CIDR block must be a /29 block; the Compute Engine
+          networks API forbids a smaller block, and using a larger block would
+          be wasteful (a node can only consume one IP address).
+
+          Errors will occur if the CIDR block has already been used for a
+          currently existing TPU node, the CIDR block conflicts with any
+          subnetworks in the user's provided network, or the provided network
+          is peered with another network that is using that CIDR block.
+      - !ruby/object:Api::Type::String
+        name: 'serviceAccount'
+        output: true
+        description: |
+          The service account used to run the tensor flow services within the
+          node. To share resources, including Google Cloud Storage data, with
+          the Tensorflow job running in the Node, this account must have
+          permissions to that data.
+      - !ruby/object:Api::Type::NestedObject
+        name: 'schedulingConfig'
+        input: true
+        description: |
+          Sets the scheduling options for this TPU instance.
+        properties:
+          - !ruby/object:Api::Type::Boolean
+            name: 'preemptible'
+            description: |
+              Defines whether the TPU instance is preemptible.
+            default_value: false
+      - !ruby/object:Api::Type::Array
+        name: 'networkEndpoints'
+        output: true
+        description: |
+          The network endpoints where TPU workers can be accessed and sent work.
+          It is recommended that Tensorflow clients of the node first reach out
+          to the first (index 0) entry.
+        item_type: !ruby/object:Api::Type::NestedObject
+          properties:
+            - !ruby/object:Api::Type::String
+              name: 'ipAddress'
+              output: true
+              description: |
+                The IP address of this network endpoint.
+            - !ruby/object:Api::Type::Integer
+              name: 'port'
+              output: true
+              description: |
+                The port of this network endpoint.
+      - !ruby/object:Api::Type::KeyValuePairs
+        name: 'labels'
+        input: true
+        description: Resource labels to represent user provided metadata.

products/tpu/terraform.yaml

@@ -0,0 +1,53 @@
+# Copyright 2018 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Provider::Terraform::Config
+overrides: !ruby/object:Overrides::ResourceOverrides
+  Node: !ruby/object:Overrides::Terraform::ResourceOverride
+    id_format: "{{project}}/{{zone}}/{{name}}"
+    import_format: ["projects/{{project}}/locations/{{zone}}/nodes/{{name}}"]
+    autogen_async: true
+    examples:
+      - !ruby/object:Provider::Terraform::Examples
+        name: "tpu_node_basic"
+        primary_resource_id: "tpu"
+        version: <%= version_name %>
+        vars:
+          node_name: "test-tpu"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "tpu_node_full"
+        primary_resource_id: "tpu"
+        version: <%= version_name %>
+        vars:
+          node_name: "test-tpu"
+          network_name: "test-tpu-network"
+    properties:
+      name: !ruby/object:Overrides::Terraform::PropertyOverride
+        custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb'
+      network: !ruby/object:Overrides::Terraform::PropertyOverride
+        default_from_api: true
+        diff_suppress_func: 'compareSelfLinkOrResourceName'
+      schedulingConfig: !ruby/object:Overrides::Terraform::PropertyOverride
+        diff_suppress_func: 'compareTpuNodeSchedulingConfig'
+      schedulingConfig.preemptible: !ruby/object:Overrides::Terraform::PropertyOverride
+        diff_suppress_func: 'compareTpuNodeSchedulingConfig'
+      zone: !ruby/object:Overrides::Terraform::PropertyOverride
+        ignore_read: true
+    custom_code: !ruby/object:Provider::Terraform::CustomCode
+      constants:  'templates/terraform/constants/tpu_node.erb'
+# This is for copying files over
+files: !ruby/object:Provider::Config::Files
+  # These files have templating (ERB) code that will be run.
+  # This is usually to add licensing info, autogeneration notices, etc.
+  compile:
+<%= lines(indent(compile('provider/terraform/product~compile.yaml'), 4)) -%>

templates/terraform/constants/tpu_node.erb

@@ -0,0 +1,13 @@
+// compareTpuNodeSchedulingConfig diff suppresses for the default
+// scheduling, i.e. if preemptible is false, the API may either return no
+// schedulingConfig or an empty schedulingConfig.
+func compareTpuNodeSchedulingConfig(k, old, new string, d *schema.ResourceData) bool {
+	if k == "scheduling_config.0.preemptible" {
+		return old == "" && new == "false"
+	}
+	if k == "scheduling_config.#" {
+		o, n := d.GetChange("scheduling_config.0.preemptible")
+		return o.(bool) == n.(bool)
+	}
+	return false
+}

templates/terraform/examples/tpu_node_basic.tf.erb

@@ -0,0 +1,13 @@
+<%#-
+	WARNING: cidr_block must not overlap with other existing TPU blocks
+	Make sure if you change this value that it does not overlap with the
+	autogenerated examples.
+-%>
+resource "google_tpu_node" "<%= ctx[:primary_resource_id] %>" {
+	name           = "<%= ctx[:vars]["node_name"] %>"
+	zone           = "us-central1-b"
+
+	accelerator_type   = "v3-8"
+	tensorflow_version = "1.9"
+	cidr_block         = "10.2.0.0/29"
+}

templates/terraform/examples/tpu_node_full.tf.erb

@@ -0,0 +1,30 @@
+resource "google_compute_network" "tpu_network" {
+	name               = "<%= ctx[:vars]["network_name"] %>"
+	auto_create_subnetworks = false
+}
+
+<%#-
+	WARNING: cidr_block must not overlap with other existing TPU blocks
+	Make sure if you change this value that it does not overlap with the
+	autogenerated examples.
+-%>
+resource "google_tpu_node" "<%= ctx[:primary_resource_id] %>" {
+	name               = "<%= ctx[:vars]["node_name"] %>"
+	zone               = "us-central1-b"
+
+	accelerator_type   = "v3-8"
+
+	cidr_block         = "10.3.0.0/29"
+	tensorflow_version = "1.12"
+
+	description = "Terraform Google Provider test TPU"
+	network = "${google_compute_network.tpu_network.name}"
+
+	labels {
+		foo = "bar"
+	}
+
+	scheduling_config {
+		preemptible = true
+	}
+}

third_party/terraform/tests/resource_tpu_node_test.go

@@ -0,0 +1,57 @@
+package google
+
+import (
+	"testing"
+
+	"fmt"
+	"github.com/hashicorp/terraform/helper/acctest"
+	"github.com/hashicorp/terraform/helper/resource"
+)
+
+func TestAccTpuNode_tpuNodeBUpdateTensorFlowVersion(t *testing.T) {
+	t.Parallel()
+
+	nodeId := acctest.RandomWithPrefix("tf-test")
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckTpuNodeDestroy,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccTpuNode_tpuNodeTensorFlow(nodeId, "1.11"),
+			},
+			{
+				ResourceName:            "google_tpu_node.tpu",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"zone"},
+			},
+			{
+				Config: testAccTpuNode_tpuNodeTensorFlow(nodeId, "1.12"),
+			},
+			{
+				ResourceName:            "google_tpu_node.tpu",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"zone"},
+			},
+		},
+	})
+}
+
+// WARNING: cidr_block must not overlap with other existing TPU blocks
+// Make sure if you change this value that it does not overlap with the
+// autogenerated examples.
+func testAccTpuNode_tpuNodeTensorFlow(nodeId, tensorFlowVer string) string {
+	return fmt.Sprintf(`
+resource "google_tpu_node" "tpu" {
+  name           = "%s"
+  zone           = "us-central1-b"
+
+  accelerator_type   = "v3-8"
+  tensorflow_version = "%s"
+  cidr_block         = "10.1.0.0/29"
+}
+`, nodeId, tensorFlowVer)
+}

third_party/terraform/utils/provider.go.erb

@@ -157,6 +157,7 @@ func ResourceMapWithErrors() (map[string]*schema.Resource, error) {
 			GeneratedSourceRepoResourcesMap,
 			GeneratedSpannerResourcesMap,
 			GeneratedStorageResourcesMap,
+			GeneratedTpuResourcesMap,
 			GeneratedMonitoringResourcesMap,
 			map[string]*schema.Resource{
 				"google_app_engine_application":                resourceAppEngineApplication(),