Merge branch 'master' into megan_bucket_retention_policy

.ci/magic-modules/generate-ansible.sh

@@ -13,10 +13,7 @@ pushd magic-modules-branched
 # Choose the author of the most recent commit as the downstream author
 COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 
-for i in $(find products/ -name 'ansible.yaml' -printf '%h\n');
-do
-  bundle exec compiler -p $i -e ansible -o "build/ansible/"
-done
+bundle exec compiler -a -e ansible -o "build/ansible/"
 
 ANSIBLE_COMMIT_MSG="$(cat .git/title)"
 

.ci/magic-modules/generate-inspec.sh

@@ -13,10 +13,7 @@ pushd magic-modules-branched
 # Choose the author of the most recent commit as the downstream author
 COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 
-for i in $(find products/ -name 'inspec.yaml' -printf '%h\n');
-do
-  bundle exec compiler -p $i -e inspec -o "build/inspec/"
-done
+bundle exec compiler -a -e inspec -o "build/inspec/"
 
 INSPEC_COMMIT_MSG="$(cat .git/title)"
 

.ci/magic-modules/pyutils/strutils.py

@@ -84,10 +84,13 @@ def set_release_note(release_note, body):
   Returns:
     Modified text
   """
-  edited = ""
-  for segment in re.split(r'`{3}releasenote', body, re.S):
-    idx = segment.find('```\n')
-    edited += segment if idx < 0 else segment[idx+3:]
+  edited = body
+  tkns = re.split(RELEASE_NOTE_SUB_RE, body, re.S)
+  if len(tkns) > 1:
+    edited = tkns[0]
+    for tkn in tkns[1:]:
+      idx = tkn.find("```")
+      edited += tkn if idx < 0 else tkn[idx+3:]
 
   release_note = release_note.strip()
   if release_note:

api/async.rb

@@ -47,13 +47,20 @@ class Operation < Api::Object
       attr_reader :wait_ms
       attr_reader :timeouts
 
+      # Use this if the resource includes the full operation url.
+      attr_reader :full_url
+
       def validate
         super
 
         check :kind, type: String
         check :path, type: String, required: true
-        check :base_url, type: String, required: true
+        check :base_url, type: String
         check :wait_ms, type: Integer, required: true
+
+        check :full_url, type: String
+
+        conflicts %i[base_url full_url]
       end
     end
 

api/resource.rb

@@ -86,12 +86,13 @@ module Properties
       # within the collection (list) json. Will default to the
       # camelcase pluralize name of the resource.
       attr_reader :collection_url_key
-      # [Optional] This is an array with items that uniquely identify the
-      # resource.
-      # This is useful in case an API returns a list result and we need
-      # to fetch the particular resource we're interested in from that
-      # list.  Otherwise, it's safe to leave empty.
-      # If empty, we assume that `name` is the identifier.
+      # [Optional] An ordered list of names of parameters that uniquely identify
+      # the resource.
+      # Generally, it's safe to leave empty, in which case it defaults to `name`.
+      # Other values are normally useful in cases where an object has a parent
+      # and is identified by some non-name value, such as an ip+port pair.
+      # If you're writing a fine-grained resource (eg with nested_query) a value
+      # must be set.
       attr_reader :identity
       # [Optional] (Api::Resource::NestedQuery) This is useful in case you need
       # to change the query made for GET requests only. In particular, this is
@@ -246,7 +247,7 @@ def identity
       if @identity.nil?
         props.select { |p| p.name == Api::Type::String::NAME.name }
       else
-        props.select { |p| @identity.include?(p.name) }
+        props.select { |p| @identity.include?(p.name) }.sort_by { |p| @identity.index p.name }
       end
     end
 

compiler.rb

@@ -34,7 +34,7 @@
 require 'provider/terraform_object_library'
 require 'pp' if ENV['COMPILER_DEBUG']
 
-product_names = nil
+products_to_compile = nil
 all_products = false
 yaml_dump = false
 output_path = nil
@@ -50,7 +50,7 @@
 # rubocop:disable Metrics/BlockLength
 OptionParser.new do |opt|
   opt.on('-p', '--product PRODUCT', Array, 'Folder[,Folder...] with product catalog') do |p|
-    product_names = p
+    products_to_compile = p
   end
   opt.on('-a', '--all', 'Build all products. Cannot be used with --product.') do
     all_products = true
@@ -86,32 +86,35 @@
 end.parse!
 # rubocop:enable Metrics/BlockLength
 
-raise 'Cannot use -p/--products and -a/--all simultaneously' if product_names && all_products
-raise 'Either -p/--products OR -a/--all must be present' if product_names.nil? && !all_products
+raise 'Cannot use -p/--products and -a/--all simultaneously' \
+  if products_to_compile && all_products
+raise 'Either -p/--products OR -a/--all must be present' \
+  if products_to_compile.nil? && !all_products
 raise 'Option -o/--output is a required parameter' if output_path.nil?
 raise 'Option -e/--engine is a required parameter' if provider_name.nil?
 
-if all_products
-  product_names = []
-  Dir['products/**/api.yaml'].each do |file_path|
-    product_names.push(File.dirname(file_path))
-  end
+all_product_files = []
+Dir['products/**/api.yaml'].each do |file_path|
+  all_product_files.push(File.dirname(file_path))
+end
 
-  if override_dir
-    Dir["#{override_dir}/products/**/api.yaml"].each do |file_path|
-      product = File.dirname(Pathname.new(file_path).relative_path_from(override_dir))
-      product_names.push(product) unless product_names.include? product
-    end
+if override_dir
+  Dir["#{override_dir}/products/**/api.yaml"].each do |file_path|
+    product = File.dirname(Pathname.new(file_path).relative_path_from(override_dir))
+    all_product_files.push(product) unless all_product_files.include? product
   end
-
-  raise 'No api.yaml files found. Check provider/engine name.' if product_names.empty?
 end
 
+products_to_compile = all_product_files if all_products
+
+raise 'No api.yaml files found. Check provider/engine name.' if products_to_compile.empty?
+
 start_time = Time.now
 
+products_for_version = []
 provider = nil
 # rubocop:disable Metrics/BlockLength
-product_names.each do |product_name|
+all_product_files.each do |product_name|
   product_override_path = ''
   provider_override_path = ''
   product_override_path = File.join(override_dir, product_name, 'api.yaml') if override_dir
@@ -143,9 +146,7 @@
   raise "Output path '#{output_path}' does not exist or is not a directory" \
     unless Dir.exist?(output_path)
 
-  Google::LOGGER.info "Compiling '#{product_name}' (at #{version}) output to '#{output_path}'"
-  Google::LOGGER.info \
-    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
+  Google::LOGGER.info "Loading '#{product_name}' (at #{version})'"
 
   product_api = Api::Compiler.new(product_yaml).run
   product_api.validate
@@ -164,8 +165,18 @@
   # Load any dynamic overrides passed in with -r
   if File.exist?(provider_override_path)
     product_api, provider_config, = \
-      Provider::Config.parse(provider_override_path, product_api, version)
+      Provider::Config.parse(provider_override_path, product_api, version, override_dir)
   end
+  products_for_version.push(product_api.name)
+
+  unless products_to_compile.include?(product_name)
+    Google::LOGGER.info "Skipping product '#{product_name}' as it was not specified to be compiled"
+    next
+  end
+
+  Google::LOGGER.info "Compiling '#{product_name}' (at #{version}) output to '#{output_path}'"
+  Google::LOGGER.info \
+    "Generating types: #{types_to_generate.empty? ? 'ALL' : types_to_generate}"
 
   pp provider_config if ENV['COMPILER_DEBUG']
 
@@ -193,6 +204,5 @@
 # of the products loop. This will get called with the provider from the final iteration
 # of the loop
 provider&.copy_common_files(output_path, version)
-provider&.compile_common_files(output_path, version)
-
+provider&.compile_common_files(output_path, version, products_for_version.sort)
 # rubocop:enable Metrics/BlockLength

google/yaml_validator.rb

@@ -82,6 +82,16 @@ def check(variable, **opts)
         unless opts[:allowed].include?(value)
     end
 
+    def conflicts(list)
+      value_checked = false
+      list.each do |item|
+        next if instance_variable_get("@#{item}").nil?
+        raise "#{list.join(',')} cannot be set at the same time" if value_checked
+
+        value_checked = true
+      end
+    end
+
     private
 
     def check_type(name, object, type)

products/accesscontextmanager/terraform.yaml

@@ -54,7 +54,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         custom_flatten: templates/terraform/custom_flatten/default_if_empty.erb
     custom_code: !ruby/object:Provider::Terraform::CustomCode
       encoder: templates/terraform/encoders/access_level_never_send_parent.go.erb
-      custom_import: templates/terraform/custom_import/access_level_self_link_as_name_and_set_parent.go.erb
+      custom_import: templates/terraform/custom_import/set_access_policy_parent_from_self_link.go.erb
   ServicePerimeter: !ruby/object:Overrides::Terraform::ResourceOverride
     timeouts: !ruby/object:Api::Timeouts
       insert_minutes: 6
@@ -79,7 +79,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         input: true
     custom_code: !ruby/object:Provider::Terraform::CustomCode
       encoder: templates/terraform/encoders/access_level_never_send_parent.go.erb
-      custom_import: templates/terraform/custom_import/access_level_self_link_as_name_and_set_parent.go.erb
+      custom_import: templates/terraform/custom_import/set_access_policy_parent_from_self_link.go.erb
 # This is for copying files over
 files: !ruby/object:Provider::Config::Files
   # These files have templating (ERB) code that will be run.

products/appengine/ansible.yaml

@@ -24,5 +24,5 @@ overrides: !ruby/object:Overrides::ResourceOverrides
     identity:
       - priority
 files: !ruby/object:Provider::Config::Files
-  compile:
-<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
+  resource:
+<%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>

products/bigquery/ansible.yaml

@@ -46,7 +46,5 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           must contain only letters (a-z, A-Z), numbers (0-9), or
           underscores. The maximum length is 1,024 characters.
 files: !ruby/object:Provider::Config::Files
-  copy:
-<%= lines(indent(compile('provider/ansible/common~copy.yaml'), 4)) -%>
-  compile:
-<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
+  resource:
+<%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>

products/binaryauthorization/api.yaml

@@ -15,6 +15,9 @@
 name: BinaryAuthorization
 display_name: Binary Authorization
 versions:
+  - !ruby/object:Api::Product::Version
+    name: ga
+    base_url: https://binaryauthorization.googleapis.com/v1/
   - !ruby/object:Api::Product::Version
     name: beta
     base_url: https://binaryauthorization.googleapis.com/v1beta1/

products/binaryauthorization/terraform.yaml

@@ -23,6 +23,9 @@ overrides: !ruby/object:Overrides::ResourceOverrides
       vars:
         attestor_name: "test-attestor"
         note_name: "test-attestor-note"
+    custom_code: !ruby/object:Provider::Terraform::CustomCode
+      encoder: templates/terraform/encoders/binauth_attestor_note_field_name.go.erb
+      decoder: templates/terraform/decoders/binauth_attestor_note_field_name.go.erb
     properties:
       name: !ruby/object:Overrides::Terraform::PropertyOverride
         custom_flatten: 'templates/terraform/custom_flatten/name_from_self_link.erb'

products/cloudbuild/ansible.yaml

@@ -29,5 +29,5 @@ overrides: !ruby/object:Overrides::ResourceOverrides
       id: !ruby/object:Overrides::Ansible::PropertyOverride
         output: false
 files: !ruby/object:Provider::Config::Files
-  compile:
-<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
+  resource:
+<%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>

products/cloudscheduler/ansible.yaml

@@ -27,5 +27,5 @@ overrides: !ruby/object:Overrides::ResourceOverrides
     provider_helpers:
       - products/cloudscheduler/helpers/python/job.py
 files: !ruby/object:Provider::Config::Files
-  compile:
-<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
+  resource:
+<%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>

products/compute/ansible.yaml

@@ -14,16 +14,14 @@
 --- !ruby/object:Provider::Ansible::Config
 # This is where custom code would be defined eventually.
 datasources: !ruby/object:Overrides::ResourceOverrides
-  Autoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   BackendBucketSignedUrlKey: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   BackendServiceSignedUrlKey: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
-  Snapshot: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   ManagedSslCertificate: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
+  RegionAutoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
+    exclude: true
   # Readonly resources.
   DiskType: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
@@ -39,8 +37,6 @@ datasources: !ruby/object:Overrides::ResourceOverrides
     exclude: true
   NodeTemplate: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
-  RegionAutoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   RegionBackendService: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   Region: !ruby/object:Overrides::Ansible::ResourceOverride
@@ -54,6 +50,26 @@ datasources: !ruby/object:Overrides::ResourceOverrides
   Zone: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
 overrides: !ruby/object:Overrides::ResourceOverrides
+  Autoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
+    properties:
+      autoscalingPolicy.minNumReplicas: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['minReplicas']
+      autoscalingPolicy.maxNumReplicas: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['maxReplicas']
+      autoscalingPolicy.coolDownPeriodSec: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['cooldownPeriod']
+      autoscalingPolicy.cpuUtilization.utilizationTarget: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['target']
+      autoscalingPolicy.customMetricUtilizations: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['metric']
+      autoscalingPolicy.customMetricUtilizations.metric: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['name']
+      autoscalingPolicy.customMetricUtilizations.utilizationTarget: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['target']
+      autoscalingPolicy.customMetricUtilizations.utilizationTargetType: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['type']
+      autoscalingPolicy.loadBalancingUtilization.utilizationTarget: !ruby/object:Overrides::Ansible::PropertyOverride
+        aliases: ['target']
   BackendService: !ruby/object:Overrides::Ansible::ResourceOverride
     properties:
       timeoutSec: !ruby/object:Overrides::Ansible::PropertyOverride
@@ -230,22 +246,18 @@ overrides: !ruby/object:Overrides::ResourceOverrides
   TargetPool: !ruby/object:Overrides::Ansible::ResourceOverride
     transport: !ruby/object:Overrides::Ansible::Transport
       encoder: encode_request
-      decoder: decode_request
+      decoder: decode_response
     provider_helpers:
       - 'products/compute/helpers/python/provider_target_pool.py'
   TargetVpnGateway: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: false
   # Not yet implemented.
-  Autoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   BackendBucketSignedUrlKey: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   BackendServiceSignedUrlKey: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
   RegionAutoscaler: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
-  Snapshot: !ruby/object:Overrides::Ansible::ResourceOverride
-    exclude: true
   # Ansible tasks must alter infrastructure.
   # This means that virtual objects are a poor fit.
   DiskType: !ruby/object:Overrides::Ansible::ResourceOverride
@@ -273,5 +285,5 @@ overrides: !ruby/object:Overrides::ResourceOverrides
   Zone: !ruby/object:Overrides::Ansible::ResourceOverride
     exclude: true
 files: !ruby/object:Provider::Config::Files
-  compile:
-<%= lines(indent(compile('provider/ansible/product~compile.yaml'), 4)) -%>
+  resource:
+<%= lines(indent(compile('provider/ansible/resource~compile.yaml'), 4)) -%>