Merge branch 'master' into ip-range-reservation

.ci/magic-modules/generate-terraform.sh

@@ -25,11 +25,9 @@ popd
 pushd magic-modules-branched
 LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 bundle install
-bundle exec compiler -p products/binaryauthorization -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/compute -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/containeranalysis -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/resourcemanager -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/redis -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
+
+# Build all terraform products
+bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
 
 # Resources that were already using beta APIs before they started being autogenerated
 bundle exec compiler -v beta -p products/compute -t Address,Firewall,ForwardingRule,GlobalAddress,RegionDisk,Subnetwork,VpnTunnel -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"

compile/core.rb

@@ -142,7 +142,7 @@ def compile(file, caller_frame = 1)
       has_erbout = ctx.local_variables.include?(:_erbout)
       content = ctx.local_variable_get(:_erbout) if has_erbout # save code
       ctx.local_variable_set(:compiler, compiler)
-      Google::LOGGER.info "Compiling #{file}"
+      Google::LOGGER.debug "Compiling #{file}"
       input = ERB.new get_helper_file(file), nil, '-%>'
       compiled = input.result(ctx)
       ctx.local_variable_set(:_erbout, content) if has_erbout # restore code

compiler.rb

@@ -36,18 +36,22 @@
 require 'provider/terraform'
 require 'pp' if ENV['COMPILER_DEBUG']
 
-product_name = nil
+product_names = nil
+all_products = false
 output_path = nil
 provider_name = nil
 types_to_generate = []
 version = nil
 
 ARGV << '-h' if ARGV.empty?
-Google::LOGGER.level = Logger::WARN
+Google::LOGGER.level = Logger::INFO
 
 OptionParser.new do |opt|
-  opt.on('-p', '--product PRODUCT', 'Folder with product catalog') do |p|
-    product_name = p
+  opt.on('-p', '--product PRODUCT', Array, 'Folder[,Folder...] with product catalog') do |p|
+    product_names = p
+  end
+  opt.on('-a', '--all', 'Build all products. Cannot be used with --product.') do
+    all_products = true
   end
   opt.on('-o', '--output OUTPUT', 'Folder for module output') do |o|
     output_path = o
@@ -66,35 +70,47 @@
     exit
   end
   opt.on('-d', '--debug', 'Show all debug logs') do |_debug|
-    Google::LOGGER.level = Logger::INFO
+    Google::LOGGER.level = Logger::DEBUG
   end
 end.parse!
 
-raise 'Option -p/--product is a required parameter' if product_name.nil?
+raise 'Cannt use -p/--products and -a/--all simultaneously' if product_names && all_products
+raise 'Either -p/--products OR -a/--all must be present' if product_names.nil? && !all_products
 raise 'Option -o/--output is a required parameter' if output_path.nil?
 raise 'Option -e/--engine is a required parameter' if provider_name.nil?
 
-product_yaml_path = File.join(product_name, 'api.yaml')
-raise "Product '#{product_name}' does not have an api.yaml file" \
-  unless File.exist?(product_yaml_path)
+if all_products
+  product_names = []
+  Dir["products/**/#{provider_name}.yaml"].each do |file_path|
+    product_names.push(File.dirname(file_path))
+  end
+
+  raise "No #{provider_name}.yaml files found. Check provider/engine name." if product_names.empty?
+end
+
+product_names.each do |product_name|
+  product_yaml_path = File.join(product_name, 'api.yaml')
+  raise "Product '#{product_name}' does not have an api.yaml file" \
+    unless File.exist?(product_yaml_path)
 
-provider_yaml_path = File.join(product_name, "#{provider_name}.yaml")
-raise "Product '#{product_name}' does not have a #{provider_name}.yaml file" \
-  unless File.exist?(provider_yaml_path)
+  provider_yaml_path = File.join(product_name, "#{provider_name}.yaml")
+  raise "Product '#{product_name}' does not have a #{provider_name}.yaml file" \
+    unless File.exist?(provider_yaml_path)
 
-raise "Output path '#{output_path}' does not exist or is not a directory" \
-  unless Dir.exist?(output_path)
+  raise "Output path '#{output_path}' does not exist or is not a directory" \
+    unless Dir.exist?(output_path)
 
-Google::LOGGER.info "Compiling '#{product_name}' output to '#{output_path}'"
-Google::LOGGER.info \
-  "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
+  Google::LOGGER.info "Compiling '#{product_name}' output to '#{output_path}'"
+  Google::LOGGER.info \
+    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
 
-product_api = Api::Compiler.new(product_yaml_path).run
-product_api.validate
-pp product_api if ENV['COMPILER_DEBUG']
+  product_api = Api::Compiler.new(product_yaml_path).run
+  product_api.validate
+  pp product_api if ENV['COMPILER_DEBUG']
 
-provider_config = Provider::Config.parse(provider_yaml_path, product_api, version)
-pp provider_config if ENV['COMPILER_DEBUG']
+  provider_config = Provider::Config.parse(provider_yaml_path, product_api, version)
+  pp provider_config if ENV['COMPILER_DEBUG']
 
-provider = provider_config.provider.new(provider_config, product_api)
-provider.generate output_path, types_to_generate, version
+  provider = provider_config.provider.new(provider_config, product_api)
+  provider.generate output_path, types_to_generate, version
+end

google/yaml_validator.rb

@@ -33,12 +33,12 @@ def allowed_classes
     end
 
     def validate
-      Google::LOGGER.info "Validating #{self.class} '#{@name}'"
+      Google::LOGGER.debug "Validating #{self.class} '#{@name}'"
       check_extraneous_properties
     end
 
     def set_variable(value, property)
-      Google::LOGGER.info "Setting variable of #{value} to #{self}"
+      Google::LOGGER.debug "Setting variable of #{value} to #{self}"
       ensure_property_does_not_exist property
       instance_variable_set("@#{property}", value)
     end
@@ -66,9 +66,9 @@ def check_type(name, object, type)
 
     def log_check_type(object)
       if object.respond_to?(:name)
-        Google::LOGGER.info "Checking object #{object.name}"
+        Google::LOGGER.debug "Checking object #{object.name}"
       else
-        Google::LOGGER.info "Checking object #{object}"
+        Google::LOGGER.debug "Checking object #{object}"
       end
     end
 
@@ -83,7 +83,7 @@ def check_optional_property(property, type = nil)
     end
 
     def check_property_value(property, prop_value, type)
-      Google::LOGGER.info "Checking '#{property}' on #{object_display_name}"
+      Google::LOGGER.debug "Checking '#{property}' on #{object_display_name}"
       raise "Missing '#{property}' on #{object_display_name}" if prop_value.nil?
       check_type property, prop_value, type unless type.nil?
       prop_value.validate if prop_value.is_a?(Api::Object)
@@ -98,7 +98,7 @@ def check_extraneous_properties
       instance_variables.each do |variable|
         var_name = variable.id2name[1..-1]
         next if var_name.start_with?('__')
-        Google::LOGGER.info "Validating '#{var_name}' on #{object_display_name}"
+        Google::LOGGER.debug "Validating '#{var_name}' on #{object_display_name}"
         raise "Extraneous variable '#{var_name}' in #{object_display_name}" \
           unless methods.include?(var_name.intern)
       end
@@ -107,9 +107,9 @@ def check_extraneous_properties
     def check_property_list(name, type = nil)
       obj_list = instance_variable_get("@#{name}")
       if obj_list.nil?
-        Google::LOGGER.info "No next level @ #{object_display_name}: #{name}"
+        Google::LOGGER.debug "No next level @ #{object_display_name}: #{name}"
       else
-        Google::LOGGER.info \
+        Google::LOGGER.debug \
           "Checking next level for #{object_display_name}: #{name}"
         obj_list.each { |o| check_property_value "#{name}:item", o, type }
       end

provider/ansible.rb

@@ -13,39 +13,18 @@
 
 require 'provider/config'
 require 'provider/core'
-require 'provider/ansible/manifest'
-require 'provider/ansible/example'
+require 'provider/ansible/config'
 require 'provider/ansible/documentation'
+require 'provider/ansible/example'
+require 'provider/ansible/manifest'
 require 'provider/ansible/module'
+require 'provider/ansible/property_override'
 require 'provider/ansible/request'
 require 'provider/ansible/resourceref'
 require 'provider/ansible/resource_override'
-require 'provider/ansible/property_override'
 
 module Provider
   module Ansible
-    # Settings for the Ansible provider
-    class Config < Provider::Config
-      attr_reader :manifest
-
-      def provider
-        Provider::Ansible::Core
-      end
-
-      def resource_override
-        Provider::Ansible::ResourceOverride
-      end
-
-      def property_override
-        Provider::Ansible::PropertyOverride
-      end
-
-      def validate
-        super
-        check_optional_property :manifest, Provider::Ansible::Manifest
-      end
-    end
-
     # Code generator for Ansible Cookbooks that manage Google Cloud Platform
     # resources.
     # TODO(alexstephen): Split up class into multiple modules.

provider/ansible/config.rb

@@ -0,0 +1,41 @@
+# Copyright 2017 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'provider/config'
+require 'provider/core'
+
+module Provider
+  module Ansible
+    # Settings for the Ansible provider
+    class Config < Provider::Config
+      attr_reader :manifest
+
+      def provider
+        Provider::Ansible::Core
+      end
+
+      def resource_override
+        Provider::Ansible::ResourceOverride
+      end
+
+      def property_override
+        Provider::Ansible::PropertyOverride
+      end
+
+      def validate
+        super
+        check_optional_property :manifest, Provider::Ansible::Manifest
+      end
+    end
+  end
+end

provider/chef.rb

@@ -14,6 +14,7 @@
 require 'google/ruby_utils'
 require 'provider/config'
 require 'provider/core'
+require 'provider/chef/config'
 require 'provider/chef/manifest'
 require 'provider/chef/property_override'
 require 'provider/chef/resource_override'
@@ -30,34 +31,6 @@ class Chef < Provider::Core
     RESERVED_WORDS = %w[deprecated updated].freeze
     TEST_FOLDER = 'recipes'.freeze
 
-    # Settings for the provider
-    class Config < Provider::Config
-      attr_reader :manifest
-      attr_reader :operating_systems
-      # TODO(alexstephen): Convert this to a regular function generator
-      # like Puppet.
-      attr_reader :functions
-
-      def provider
-        Provider::Chef
-      end
-
-      def resource_override
-        Provider::Chef::ResourceOverride
-      end
-
-      def property_override
-        Provider::Chef::PropertyOverride
-      end
-
-      def validate
-        super
-        check_optional_property :manifest, Provider::Chef::Manifest
-        check_property_list \
-          :operating_systems, Provider::Config::OperatingSystem
-      end
-    end
-
     # A custom client side function for Chef
     class Function < Provider::Config::Function
       attr_reader :search_paths

provider/chef/config.rb

@@ -0,0 +1,45 @@
+# Copyright 2017 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'provider/config'
+
+module Provider
+  class Chef < Provider::Core
+    # Settings for the provider
+    class Config < Provider::Config
+      attr_reader :manifest
+      attr_reader :operating_systems
+      # TODO(alexstephen): Convert this to a regular function generator like Puppet.
+      attr_reader :functions
+
+      def provider
+        Provider::Chef
+      end
+
+      def resource_override
+        Provider::Chef::ResourceOverride
+      end
+
+      def property_override
+        Provider::Chef::PropertyOverride
+      end
+
+      def validate
+        super
+        check_optional_property :manifest, Provider::Chef::Manifest
+        check_property_list \
+          :operating_systems, Provider::Config::OperatingSystem
+      end
+    end
+  end
+end

provider/core.rb

@@ -96,7 +96,7 @@ def copy_files(output_folder)
         target_file = File.join(output_folder, target)
         target_dir = File.dirname(target_file)
         @sourced << relative_path(target_file, output_folder)
-        Google::LOGGER.info "Copying #{source} => #{target}"
+        Google::LOGGER.debug "Copying #{source} => #{target}"
         FileUtils.mkpath target_dir unless Dir.exist?(target_dir)
         FileUtils.copy_entry source, target_file
       end
@@ -183,7 +183,7 @@ def list_manual_network_data
     # rubocop:disable Metrics/AbcSize
     def compile_file_list(output_folder, files, data = {})
       files.each do |target, source|
-        Google::LOGGER.info "Compiling #{source} => #{target}"
+        Google::LOGGER.debug "Compiling #{source} => #{target}"
         target_file = File.join(output_folder, target)
                           .gsub('{{product_name}}', @api.prefix[1..-1])
 
@@ -632,7 +632,7 @@ def generate_file(data)
 
     def generate_file_write(ctx, data)
       enforce_file_expectations data[:out_file] do
-        Google::LOGGER.info "Generating #{data[:name]} #{data[:type]}"
+        Google::LOGGER.debug "Generating #{data[:name]} #{data[:type]}"
         write_file data[:out_file], compile_file(ctx, data[:template])
       end
     end