Allow the compiler to build multiple products (#462)

<!-- A summary of the changes in this commit goes here -->
Changes that allow building multiple different products for a single
provider in a single command. 

Allows -p/--product to take an array of products

Also introduces a -a/--all flag to be used instead of
the products parameter.

<!--
Changes per downstream repository.  For each repository that you
expect to have changed, find the [tag] and write your commit
message beneath it.  More-specific tags replace less-specific tags.
For example, if you provide a message under [all], a message under
[puppet], and a message under [puppet-dns], the Terraform repository
will have the resulting commit made using the [all] message, the
Puppet Compute repository will have its commit made using [puppet],
and the Puppet DNS repository will have its commit made using
[puppet-dns].  You can delete unused tags, but you don't need to.

The structure of the PR body is important to our CI system!
The comments can be deleted, but if you want to make the downstream
commits sensible, you'll need to leave the dashed line separating
this PR's changes from the commit messages for downstream commits.
-->

-----------------------------------------------------------------

.ci/magic-modules/generate-terraform.sh

@@ -25,11 +25,9 @@ popd
 pushd magic-modules-branched
 LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 bundle install
-bundle exec compiler -p products/binaryauthorization -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/compute -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/containeranalysis -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/resourcemanager -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
-bundle exec compiler -p products/redis -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
+
+# Build all terraform products
+bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
 
 # Resources that were already using beta APIs before they started being autogenerated
 bundle exec compiler -v beta -p products/compute -t Address,Firewall,ForwardingRule,GlobalAddress,RegionDisk,Subnetwork,VpnTunnel -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"

compiler.rb

@@ -36,7 +36,8 @@
 require 'provider/terraform'
 require 'pp' if ENV['COMPILER_DEBUG']
 
-product_name = nil
+product_names = nil
+all_products = false
 output_path = nil
 provider_name = nil
 types_to_generate = []
@@ -46,8 +47,11 @@
 Google::LOGGER.level = Logger::INFO
 
 OptionParser.new do |opt|
-  opt.on('-p', '--product PRODUCT', 'Folder with product catalog') do |p|
-    product_name = p
+  opt.on('-p', '--product PRODUCT', Array, 'Folder[,Folder...] with product catalog') do |p|
+    product_names = p
+  end
+  opt.on('-a', '--all', 'Build all products. Cannot be used with --product.') do
+    all_products = true
   end
   opt.on('-o', '--output OUTPUT', 'Folder for module output') do |o|
     output_path = o
@@ -70,31 +74,43 @@
   end
 end.parse!
 
-raise 'Option -p/--product is a required parameter' if product_name.nil?
+raise 'Cannt use -p/--products and -a/--all simultaneously' if product_names && all_products
+raise 'Either -p/--products OR -a/--all must be present' if product_names.nil? && !all_products
 raise 'Option -o/--output is a required parameter' if output_path.nil?
 raise 'Option -e/--engine is a required parameter' if provider_name.nil?
 
-product_yaml_path = File.join(product_name, 'api.yaml')
-raise "Product '#{product_name}' does not have an api.yaml file" \
-  unless File.exist?(product_yaml_path)
+if all_products
+  product_names = []
+  Dir["products/**/#{provider_name}.yaml"].each do |file_path|
+    product_names.push(File.dirname(file_path))
+  end
+
+  raise "No #{provider_name}.yaml files found. Check provider/engine name." if product_names.empty?
+end
+
+product_names.each do |product_name|
+  product_yaml_path = File.join(product_name, 'api.yaml')
+  raise "Product '#{product_name}' does not have an api.yaml file" \
+    unless File.exist?(product_yaml_path)
 
-provider_yaml_path = File.join(product_name, "#{provider_name}.yaml")
-raise "Product '#{product_name}' does not have a #{provider_name}.yaml file" \
-  unless File.exist?(provider_yaml_path)
+  provider_yaml_path = File.join(product_name, "#{provider_name}.yaml")
+  raise "Product '#{product_name}' does not have a #{provider_name}.yaml file" \
+    unless File.exist?(provider_yaml_path)
 
-raise "Output path '#{output_path}' does not exist or is not a directory" \
-  unless Dir.exist?(output_path)
+  raise "Output path '#{output_path}' does not exist or is not a directory" \
+    unless Dir.exist?(output_path)
 
-Google::LOGGER.info "Compiling '#{product_name}' output to '#{output_path}'"
-Google::LOGGER.info \
-  "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
+  Google::LOGGER.info "Compiling '#{product_name}' output to '#{output_path}'"
+  Google::LOGGER.info \
+    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
 
-product_api = Api::Compiler.new(product_yaml_path).run
-product_api.validate
-pp product_api if ENV['COMPILER_DEBUG']
+  product_api = Api::Compiler.new(product_yaml_path).run
+  product_api.validate
+  pp product_api if ENV['COMPILER_DEBUG']
 
-provider_config = Provider::Config.parse(provider_yaml_path, product_api, version)
-pp provider_config if ENV['COMPILER_DEBUG']
+  provider_config = Provider::Config.parse(provider_yaml_path, product_api, version)
+  pp provider_config if ENV['COMPILER_DEBUG']
 
-provider = provider_config.provider.new(provider_config, product_api)
-provider.generate output_path, types_to_generate, version
+  provider = provider_config.provider.new(provider_config, product_api)
+  provider.generate output_path, types_to_generate, version
+end