From 2f3a1e374f6b4f3a8d16317ed20cd8729cb77112 Mon Sep 17 00:00:00 2001 From: Nathan McKinley Date: Fri, 21 Sep 2018 18:12:04 -0700 Subject: [PATCH] Generate Filestore in Terraform in Magic Modules. (#482) Merged PR #482. --- api/type.rb | 10 +- build/terraform | 2 +- products/filestore/api.yaml | 170 ++++++++++++++++++ products/filestore/terraform.yaml | 74 ++++++++ products/redis/api.yaml | 1 + products/redis/terraform.yaml | 2 - templates/terraform/constants/filestore.erb | 4 + .../terraform/decoders/redis_instance.erb | 21 --- .../terraform/encoders/redis_instance.erb | 16 -- templates/terraform/filestore_operation.go | 71 ++++++++ .../terraform/flatten_property_method.erb | 2 +- .../terraform/pre_update/update_mask.erb | 21 +++ templates/terraform/resource.erb | 2 +- 13 files changed, 352 insertions(+), 44 deletions(-) create mode 100644 products/filestore/api.yaml create mode 100644 products/filestore/terraform.yaml create mode 100644 templates/terraform/constants/filestore.erb delete mode 100644 templates/terraform/decoders/redis_instance.erb delete mode 100644 templates/terraform/encoders/redis_instance.erb create mode 100644 templates/terraform/filestore_operation.go create mode 100644 templates/terraform/pre_update/update_mask.erb diff --git a/api/type.rb b/api/type.rb index ca098427e217..5fed06c850cd 100644 --- a/api/type.rb +++ b/api/type.rb @@ -29,6 +29,7 @@ module Fields attr_reader :output # If set value will not be sent to server on sync attr_reader :input # If set to true value is used only on creation + attr_reader :url_param_only # If, true will not be send in request body attr_reader :required attr_reader :update_verb attr_reader :update_url @@ -60,6 +61,7 @@ def validate check_optional_property :output, :boolean check_optional_property :required, :boolean + check_optional_property :url_param_only, :boolean raise 'Property cannot be output and required at the same time.' \ if @output && @required @@ -250,6 +252,7 @@ class Array < Composite NESTED_ARRAY_TYPE = [Api::Type::Array, Api::Type::NestedObject].freeze RREF_ARRAY_TYPE = [Api::Type::Array, Api::Type::ResourceRef].freeze + # rubocop:disable Metrics/CyclomaticComplexity def validate super if @item_type.is_a?(NestedObject) || @item_type.is_a?(ResourceRef) @@ -257,15 +260,16 @@ def validate @item_type.set_variable(@__resource, :__resource) @item_type.set_variable(self, :__parent) end - check_property :item_type, [::String, NestedObject, ResourceRef] + check_property :item_type, [::String, NestedObject, ResourceRef, Enum] unless @item_type.is_a?(NestedObject) || @item_type.is_a?(ResourceRef) \ - || type?(@item_type) + || @item_type.is_a?(Enum) || type?(@item_type) raise "Invalid type #{@item_type}" end check_optional_property :min_size, ::Integer check_optional_property :max_size, ::Integer end + # rubocop:enable Metrics/CyclomaticComplexity def item_type_class return Api::Type::NestedObject if @item_type.is_a? NestedObject @@ -276,6 +280,8 @@ def item_type_class def property_class if @item_type.is_a?(NestedObject) || @item_type.is_a?(ResourceRef) type = @item_type.property_class + elsif @item_type.is_a?(Enum) + raise 'aaaa' else type = property_ns_prefix type << get_type(@item_type).new(@name).type diff --git a/build/terraform b/build/terraform index 042a2f57544c..c39db1e1716d 160000 --- a/build/terraform +++ b/build/terraform @@ -1 +1 @@ -Subproject commit 042a2f57544cccd5f65a25e1c18a4730a7e2e465 +Subproject commit c39db1e1716d724265a51e64d73afb16a6ebf590 diff --git a/products/filestore/api.yaml b/products/filestore/api.yaml new file mode 100644 index 000000000000..d19a90792d3b --- /dev/null +++ b/products/filestore/api.yaml @@ -0,0 +1,170 @@ +# Copyright 2018 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Product +name: Google Cloud Filestore +# There is a problem here - the generated api is called 'file', and that's +# a bad name for the library. So we set the name to gfilestore, and +# that means that Terraform in particular is going to try to import +# 'filestore'. But the library is called 'file', so instead we need to +# include a small hack to rename the library - see +# templates/terraform/constants/filestore.erb. +prefix: gfilestore +versions: + - !ruby/object:Api::Product::Version + name: beta + base_url: https://file.googleapis.com/v1beta1/ +scopes: + - https://www.googleapis.com/auth/cloud-platform +objects: + - !ruby/object:Api::Resource + name: 'Instance' + exclude: true + base_url: | + projects/{{project}}/locations/{{zone}}/instances?instanceId={{name}} + self_link: projects/{{project}}/locations/{{zone}}/instances/{{name}} + update_verb: :PATCH + description: | + A Google Cloud Filestore instance. + references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': + 'https://cloud.google.com/filestore/docs/creating-instances' + 'Use with Kubernetes': + 'https://cloud.google.com/filestore/docs/accessing-fileshares' + 'Copying Data In/Out': + 'https://cloud.google.com/filestore/docs/copying-data' + api: 'https://cloud.google.com/filestore/docs/reference/rest/v1beta1/projects.locations.instances/create' +<%= + indent(compile_file({timeouts: { insert_sec: 5 * 60 }}, + 'templates/zonal_async.yaml.erb'), 4) +%> + parameters: + - !ruby/object:Api::Type::String + name: 'zone' + description: | + The name of the Filestore zone of the instance. + required: true + input: true + url_param_only: true + properties: + - !ruby/object:Api::Type::String + name: 'name' + description: | + The resource name of the instance. + required: true + url_param_only: true + - !ruby/object:Api::Type::String + name: 'description' + description: | + A description of the instance. + - !ruby/object:Api::Type::String + name: 'state' + description: | + The instance state - short description. + output: true + exclude: true + - !ruby/object:Api::Type::String + name: 'statusMessage' + description: | + Additional information about the instance state, if available. + output: true + exclude: true + - !ruby/object:Api::Type::Time + name: 'createTime' + description: Creation timestamp in RFC3339 text format. + output: true + - !ruby/object:Api::Type::Enum + name: 'tier' + description: | + The service tier of the instance. + required: true + input: true + values: + - TIER_UNSPECIFIED + - STANDARD + - PREMIUM + - !ruby/object:Api::Type::NameValues + name: 'labels' + description: | + Resource labels to represent user-provided metadata. + key_type: Api::Type::String + value_type: Api::Type::String + - !ruby/object:Api::Type::Array + name: 'fileShares' + required: true + description: | + File system shares on the instance. For this version, only a + single file share is supported. + max_size: 1 + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'name' + description: | + The name of the fileshare (16 characters or less) + required: true + input: true + - !ruby/object:Api::Type::Integer + name: 'capacityGb' + description: | + File share capacity in GB. + required: true + - !ruby/object:Api::Type::Array + name: 'networks' + description: | + VPC networks to which the instance is connected. For this version, + only a single network is supported. + required: true + min_size: 1 + input: true + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'network' + description: | + The name of the GCE VPC network to which the + instance is connected. + required: true + input: true + - !ruby/object:Api::Type::Array + name: 'modes' + description: | + IP versions for which the instance has + IP addresses assigned. + required: true + input: true + item_type: !ruby/object:Api::Type::Enum + name: 'mode' + description: An IP version. + values: + - ADDRESS_MODE_UNSPECIFIED + - MODE_IPV4 + - MODE_IPV6 + - !ruby/object:Api::Type::String + name: 'reservedIpRange' + description: | + A /29 CIDR block that identifies the range of IP + addresses reserved for this instance. + - !ruby/object:Api::Type::Array + name: 'ipAddresses' + description: | + A list of IPv4 or IPv6 addresses. + output: true + item_type: Api::Type::String + - !ruby/object:Api::Type::String + name: 'etag' + description: | + Server-specified ETag for the instance resource to prevent + simultaneous updates from overwriting each other. + output: true diff --git a/products/filestore/terraform.yaml b/products/filestore/terraform.yaml new file mode 100644 index 000000000000..107af1288944 --- /dev/null +++ b/products/filestore/terraform.yaml @@ -0,0 +1,74 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Provider::Terraform::Config +overrides: !ruby/object:Provider::ResourceOverrides + Instance: !ruby/object:Provider::Terraform::ResourceOverride + exclude: false + id_format: "{{project}}/{{zone}}/{{name}}" + import_format: ["projects/{{project}}/locations/{{zone}}/instances/{{name}}"] + examples: | + ### Basic Usage + ```hcl + resource "google_file_instance" "instance" { + name = "test-instance" + zone = "us-central1-b" + file_shares { + capacity_gb = 2660 + name = "share1" + } + networks { + network = "default" + modes = ["MODE_IPV4"] + } + tier = "PREMIUM" + } + ``` + properties: + name: !ruby/object:Provider::Terraform::PropertyOverride + custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb' + zone: !ruby/object:Provider::Terraform::PropertyOverride + ignore_read: true + networks.reservedIpRange: !ruby/object:Provider::Terraform::PropertyOverride + default_from_api: true + custom_code: !ruby/object:Provider::Terraform::CustomCode + pre_update: templates/terraform/pre_update/update_mask.erb + constants: templates/terraform/constants/filestore.erb + +# This is for a list of example files. +examples: !ruby/object:Api::Resource::HashArray + +# This is for copying files over +files: !ruby/object:Provider::Config::Files + # All of these files will be copied verbatim. + copy: + 'google/transport.go': 'templates/terraform/transport.go' + 'google/transport_test.go': 'templates/terraform/transport_test.go' + 'google/import.go': 'templates/terraform/import.go' + 'google/import_test.go': 'templates/terraform/import_test.go' + 'google/filestore_operation.go': 'templates/terraform/filestore_operation.go' + # These files have templating (ERB) code that will be run. + # This is usually to add licensing info, autogeneration notices, etc. + compile: + 'google/provider_{{product_name}}_gen.go': 'templates/terraform/provider_gen.erb' + +# This is for custom testing code. All of our tests follow a specific pattern +# that sometimes needs to be deviated from. We're working towards a world where +# these handwritten tests would be unnecessary in many cases (custom types). +tests: !ruby/object:Api::Resource::HashArray + +# This would be for custom network responses. Tests work by running some block +# of autogenerated code and then verifying the network calls. +# The network call verifications are automatically generated, but can be +# overriden. +test_data: !ruby/object:Provider::Config::TestData diff --git a/products/redis/api.yaml b/products/redis/api.yaml index e48cc5e3cd0a..628b0e81b990 100644 --- a/products/redis/api.yaml +++ b/products/redis/api.yaml @@ -47,6 +47,7 @@ objects: The name of the Redis region of the instance. required: true input: true + url_param_only: true properties: - !ruby/object:Api::Type::String name: alternativeLocationId diff --git a/products/redis/terraform.yaml b/products/redis/terraform.yaml index f4055803f9d6..3011a65cf6e5 100644 --- a/products/redis/terraform.yaml +++ b/products/redis/terraform.yaml @@ -18,8 +18,6 @@ overrides: !ruby/object:Provider::ResourceOverrides id_format: "{{project}}/{{region}}/{{name}}" import_format: ["projects/{{project}}/locations/{{region}}/instances/{{name}}"] custom_code: !ruby/object:Provider::Terraform::CustomCode - decoder: 'templates/terraform/decoders/redis_instance.erb' - encoder: 'templates/terraform/encoders/redis_instance.erb' pre_update: 'templates/terraform/pre_update/redis_instance.erb' examples: | ### Basic Usage diff --git a/templates/terraform/constants/filestore.erb b/templates/terraform/constants/filestore.erb new file mode 100644 index 000000000000..dd35de3fd195 --- /dev/null +++ b/templates/terraform/constants/filestore.erb @@ -0,0 +1,4 @@ +<%# We have to change the name of 'file' to 'filestore' for magic-modules generation reasons.-%> +import ( + filestore "google.golang.org/api/file/v1beta1" +) diff --git a/templates/terraform/decoders/redis_instance.erb b/templates/terraform/decoders/redis_instance.erb deleted file mode 100644 index 75455f8039d2..000000000000 --- a/templates/terraform/decoders/redis_instance.erb +++ /dev/null @@ -1,21 +0,0 @@ -<%# The license inside this block applies to this file. - # Copyright 2017 Google Inc. - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. --%> -config := meta.(*Config) -region, err := getRegion(d, config) -if err != nil { - return nil, err -} -res["region"] = region -return res, nil diff --git a/templates/terraform/encoders/redis_instance.erb b/templates/terraform/encoders/redis_instance.erb deleted file mode 100644 index 218b5776b993..000000000000 --- a/templates/terraform/encoders/redis_instance.erb +++ /dev/null @@ -1,16 +0,0 @@ -<%# The license inside this block applies to this file. - # Copyright 2017 Google Inc. - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. --%> -delete(obj, "region") -return obj, nil diff --git a/templates/terraform/filestore_operation.go b/templates/terraform/filestore_operation.go new file mode 100644 index 000000000000..05ce0592d19d --- /dev/null +++ b/templates/terraform/filestore_operation.go @@ -0,0 +1,71 @@ +package google + +import ( + "fmt" + "log" + "time" + + "github.com/hashicorp/terraform/helper/resource" + file "google.golang.org/api/file/v1beta1" +) + +type FilestoreOperationWaiter struct { + Service *file.ProjectsLocationsService + Op *file.Operation +} + +func (w *FilestoreOperationWaiter) RefreshFunc() resource.StateRefreshFunc { + return func() (interface{}, string, error) { + op, err := w.Service.Operations.Get(w.Op.Name).Do() + + if err != nil { + return nil, "", err + } + + log.Printf("[DEBUG] Got %v while polling for operation %s's 'done' status", op.Done, w.Op.Name) + + return op, fmt.Sprint(op.Done), nil + } +} + +func (w *FilestoreOperationWaiter) Conf() *resource.StateChangeConf { + return &resource.StateChangeConf{ + Pending: []string{"false"}, + Target: []string{"true"}, + Refresh: w.RefreshFunc(), + } +} + +func filestoreOperationWait(service *file.Service, op *file.Operation, project, activity string) error { + return filestoreOperationWaitTime(service, op, project, activity, 4) +} + +func filestoreOperationWaitTime(service *file.Service, op *file.Operation, project, activity string, timeoutMin int) error { + if op.Done { + if op.Error != nil { + return fmt.Errorf("Error code %v, message: %s", op.Error.Code, op.Error.Message) + } + return nil + } + + w := &FilestoreOperationWaiter{ + Service: service.Projects.Locations, + Op: op, + } + + state := w.Conf() + state.Delay = 10 * time.Second + state.Timeout = time.Duration(timeoutMin) * time.Minute + state.MinTimeout = 2 * time.Second + opRaw, err := state.WaitForState() + if err != nil { + return fmt.Errorf("Error waiting for %s: %s", activity, err) + } + + op = opRaw.(*file.Operation) + if op.Error != nil { + return fmt.Errorf("Error code %v, message: %s", op.Error.Code, op.Error.Message) + } + + return nil +} diff --git a/templates/terraform/flatten_property_method.erb b/templates/terraform/flatten_property_method.erb index de5446e9a0fe..0c2ae58acb33 100644 --- a/templates/terraform/flatten_property_method.erb +++ b/templates/terraform/flatten_property_method.erb @@ -39,7 +39,7 @@ func flatten<%= prefix -%><%= titlelize_property(property) -%>(v interface{}) in for _, raw := range l { original := raw.(map[string]interface{}) transformed = append(transformed, map[string]interface{}{ - <% property.item_type.properties.each do |prop| -%> + <% property.item_type.properties.reject(&:ignore_read).each do |prop| -%> "<%= prop.name.underscore -%>": flatten<%= prefix -%><%= titlelize_property(property) -%><%= titlelize_property(prop) -%>(original["<%= prop.api_name -%>"]), <% end -%> }) diff --git a/templates/terraform/pre_update/update_mask.erb b/templates/terraform/pre_update/update_mask.erb new file mode 100644 index 000000000000..20a66b2a3235 --- /dev/null +++ b/templates/terraform/pre_update/update_mask.erb @@ -0,0 +1,21 @@ +updateMask := []string{} + +<% settable_properties.each do |prop| -%> +<%# UpdateMask documentation is not not obvious about which fields are supported or + how deeply nesting is supported. For instance, if we change the field foo.bar.baz, + it seems that *sometimes*, 'foo' is a valid value. Other times, it needs to be + 'foo.bar', and other times 'foo.bar.baz'. For now, this only works on top-level + fields - this has not been a problem yet. It might be someday! Consider modeling + the nesting off of what you see in `schema_property.erb` - that nests arbitrarily + far. +-#%> +if d.HasChange("<%= prop.name.underscore -%>") { + updateMask = append(updateMask, "<%= prop.api_name -%>") +} +<% end -%> +// updateMask is a URL parameter but not present in the schema, so replaceVars +// won't set it +url, err = addQueryParams(url, map[string]string{"updateMask": strings.Join(updateMask, ",")}) +if err != nil { + return err +} diff --git a/templates/terraform/resource.erb b/templates/terraform/resource.erb index 2a7e4e5b0803..e17363708b69 100644 --- a/templates/terraform/resource.erb +++ b/templates/terraform/resource.erb @@ -21,7 +21,7 @@ package google <% resource_name = product_ns + object.name properties = object.all_user_properties - settable_properties = properties.reject(&:output) + settable_properties = properties.reject(&:output).reject(&:url_param_only) api_name_lower = String.new(product_ns) api_name_lower[0] = api_name_lower[0].downcase has_project = object.base_url.include?("{{project}}")