add terraform-provider-google-beta to ci

.ci/ci.yml.tmpl

@@ -44,11 +44,13 @@ resources:
           authorship_restriction: true
           no_label: automerged
 
-    - name: terraform-intermediate
+{% for v in vars.terraform_v.itervalues() %}
+    - name: {{ v.short_name }}-intermediate
       type: git-branch
       source:
-          uri: [email protected]:((github-account.username))/terraform-provider-google.git
+          uri: [email protected]:((github-account.username))/{{ v.provider_name }}.git
           private_key: ((repo-key.private_key))
+{% endfor %}
 
     - name: ansible-intermediate
       type: git-branch
@@ -136,18 +138,25 @@ jobs:
                 status: pending
                 path: magic-modules
           - aggregate:
+            {% for k, v in vars.terraform_v.iteritems() %}
             - do:
                 # consumes: magic-modules-branched
                 # produces: terraform-generated
-              - task: generate-terraform
+              - task: generate-{{v.short_name}}
                 file: magic-modules-branched/.ci/magic-modules/generate-terraform.yml
-              # Puts 'terraform-generated' into the robot's fork.
-              - put: terraform-intermediate
                 params:
-                    repository: terraform-generated
-                    branch_file: magic-modules-branched/branchname
-                    only_if_diff: true
-                    force: true
+                  VERSION: {{k}}
+                  PROVIDER_NAME: {{v.provider_name}}
+                  SHORT_NAME: {{v.short_name}}
+              # Puts 'terraform-generated' into the robot's fork.
+              - aggregate:
+                - put: {{v.short_name}}-intermediate
+                  params:
+                      repository: terraform-generated/{{k}}
+                      branch_file: magic-modules-branched/branchname
+                      only_if_diff: true
+                      force: true
+            {% endfor %}
             - do:
                 # consumes: magic-modules-branched
                 # produces: ansible-generated
@@ -226,6 +235,7 @@ jobs:
                 CREDS: ((repo-key.private_key))
                 PUPPET_MODULES: {{','.join(vars.puppet_modules)}}
                 CHEF_MODULES: {{','.join(vars.chef_modules)}}
+                TERRAFORM_VERSIONS: "{{','.join(vars.terraform_properties_serialized)}}"
                 TERRAFORM_ENABLED: true
                 ANSIBLE_ENABLED: true
                 INSPEC_ENABLED: true
@@ -243,11 +253,18 @@ jobs:
           version: every
           trigger: true
           params:
-              submodules: [build/terraform]
+              submodules: [{{','.join(vars.terraform_submodules)}}]
           passed: [mm-generate]
-        - task: test
-          file: magic-modules/.ci/unit-tests/task.yml
-          timeout: 30m
+        - aggregate:
+          {% for v in vars.terraform_v.itervalues() %}
+          - task: test-{{v.short_name}}
+            file: magic-modules/.ci/unit-tests/task.yml
+            timeout: 30m
+            params:
+              PROVIDER_NAME: {{v.provider_name}}
+              SHORT_NAME: {{v.short_name}}
+              TEST_DIR: {{v.test_dir}}
+          {% endfor %}
           on_failure:
               do:
               - get: magic-modules-new-prs
@@ -407,6 +424,7 @@ jobs:
                 CHEF_REPO_USER: GoogleCloudPlatform
                 CHEF_MODULES: {{','.join(vars.chef_modules)}}
                 {%- endif %}
+                TERRAFORM_VERSIONS: "{{','.join(vars.terraform_properties_serialized)}}"
             on_failure:
               put: magic-modules-new-prs
               params:
@@ -420,9 +438,10 @@ jobs:
           # the pipeline (when a PR needs to be updated), this does that updating by pushing
           # the new code to the repository/branch from which a pull request is already open.
           - aggregate:
-            - put: terraform-intermediate
+            {% for v in vars.terraform_v.itervalues() %}
+            - put: {{v.short_name}}-intermediate
               params:
-                repository: magic-modules-with-comment/build/terraform
+                repository: magic-modules-with-comment/build/{{ v.short_name }}
                 branch_file: magic-modules-with-comment/original_pr_branch_name
                 # Every time a change runs through this pipeline, it will generate a commit with
                 # a different hash - the hash includes timestamps.  Therefore, even if there's no
@@ -432,6 +451,7 @@ jobs:
                 # not push the update even though the commit hashes are different.
                 only_if_diff: true
                 force: true
+            {% endfor %}
             - put: ansible-intermediate
               params:
                 repository: magic-modules-with-comment/build/ansible

.ci/magic-modules/create-pr.sh

@@ -26,6 +26,8 @@ echo "$ORIGINAL_PR_BRANCH" > ./original_pr_branch_name
 BRANCH_NAME="$(git config -f .gitmodules --get submodule.build/terraform.branch)"
 IFS="," read -ra PUPPET_PRODUCTS <<< "$PUPPET_MODULES"
 IFS="," read -ra CHEF_PRODUCTS <<< "$CHEF_MODULES"
+IFS="," read -ra TERRAFORM_VERSIONS <<< "$TERRAFORM_VERSIONS"
+
 git checkout -b "$BRANCH_NAME"
 
 if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then
@@ -34,22 +36,28 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then
   # There is no existing PR - this is the first pass through the pipeline and
   # we will need to create a PR using 'hub'.
   if [ -n "$TERRAFORM_REPO_USER" ]; then
-    pushd build/terraform
-
-    git log -1 --pretty=%B > ./downstream_body
-    echo "" >> ./downstream_body
-    echo "<!-- This change is generated by MagicModules. -->" >> ./downstream_body
-    if [ -n "$ORIGINAL_PR_USER" ]; then
-      echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body
-    fi
-
-    git checkout -b "$BRANCH_NAME"
-    if TF_PR=$(hub pull-request -b "$TERRAFORM_REPO_USER/terraform-provider-google:2.0.0" -F ./downstream_body); then
-      DEPENDENCIES="${DEPENDENCIES}depends: $TF_PR ${NEWLINE}"
-    else
-      echo "Terraform - did not generate a PR."
-    fi
-    popd
+    for VERSION in "${TERRAFORM_VERSIONS[@]}"; do
+      IFS=":" read -ra TERRAFORM_DATA <<< "$VERSION"
+      PROVIDER_NAME="${TERRAFORM_DATA[0]}"
+      SUBMODULE_DIR="${TERRAFORM_DATA[1]}"
+
+      pushd build/$SUBMODULE_DIR
+
+      git log -1 --pretty=%B > ./downstream_body
+      echo "" >> ./downstream_body
+      echo "<!-- This change is generated by MagicModules. -->" >> ./downstream_body
+      if [ -n "$ORIGINAL_PR_USER" ]; then
+        echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body
+      fi
+
+      git checkout -b "$BRANCH_NAME"
+      if TF_PR=$(hub pull-request -b "$TERRAFORM_REPO_USER/$PROVIDER_NAME:2.0.0" -F ./downstream_body); then
+        DEPENDENCIES="${DEPENDENCIES}depends: $TF_PR ${NEWLINE}"
+      else
+        echo "$SUBMODULE_DIR - did not generate a PR."
+      fi
+      popd
+    done
   fi
 
   if [ -n "$ANSIBLE_REPO_USER" ]; then
@@ -82,7 +90,7 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then
     fi
 
     git checkout -b "$BRANCH_NAME"
-    if INSPEC_PR=$(hub pull-request -b "$INSPEC_REPO_USER/inspec:master" -F ./downstream_body); then
+    if INSPEC_PR=$(hub pull-request -b "$INSPEC_REPO_USER/inspec-gcp:master" -F ./downstream_body); then
       DEPENDENCIES="${DEPENDENCIES}depends: $INSPEC_PR ${NEWLINE}"
     else
       echo "InSpec - did not generate a PR."
@@ -163,9 +171,14 @@ else
   git branch -f "$ORIGINAL_PR_BRANCH"
 
   if [ -n "$TERRAFORM_REPO_USER" ]; then
-    pushd build/terraform
-    git branch -f "$ORIGINAL_PR_BRANCH"
-    popd
+    for VERSION in "${TERRAFORM_VERSIONS[@]}"; do
+      IFS=":" read -ra TERRAFORM_DATA <<< "$VERSION"
+      PROVIDER_NAME="${TERRAFORM_DATA[0]}"
+      SUBMODULE_DIR="${TERRAFORM_DATA[1]}"
+      pushd "build/$SUBMODULE_DIR"
+      git branch -f "$ORIGINAL_PR_BRANCH"
+      popd
+    done
   fi
 
   for PRD in "${PUPPET_PRODUCTS[@]}"; do

.ci/magic-modules/create-pr.yml

@@ -29,3 +29,4 @@ params:
   PUPPET_MODULES: ""
   CHEF_REPO_USER: ""
   CHEF_MODULES: ""
+  TERRAFORM_VERSIONS: ""

.ci/magic-modules/generate-terraform.sh

@@ -15,10 +15,10 @@ export GOPATH="${PWD}/go"
 mkdir -p "${GOPATH}/src/github.com/terraform-providers"
 
 pushd magic-modules-branched
-ln -s "${PWD}/build/terraform/" "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google"
+ln -s "${PWD}/build/$SHORT_NAME/" "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 popd
 
-pushd "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google"
+pushd "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 go get
 popd
 
@@ -27,17 +27,17 @@ LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
 bundle install
 
 # Build all terraform products
-bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google/"
+bundle exec compiler -a -e terraform -o "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME/" -v $VERSION
 
 # This command can crash - if that happens, the script should not fail.
 set +e
-TERRAFORM_COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag terraform < .git/body)"
+TERRAFORM_COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag $SHORT_NAME < .git/body)"
 set -e
 if [ -z "$TERRAFORM_COMMIT_MSG" ]; then
   TERRAFORM_COMMIT_MSG="Magic Modules changes."
 fi
 
-pushd "build/terraform"
+pushd "build/$SHORT_NAME"
 # These config entries will set the "committer".
 git config --global user.email "[email protected]"
 git config --global user.name "Modular Magician"
@@ -47,9 +47,9 @@ git add -A
 git commit -m "$TERRAFORM_COMMIT_MSG" --author="$LAST_COMMIT_AUTHOR" || true  # don't crash if no changes
 git checkout -B "$(cat ../../branchname)"
 
-apply_patches "$PATCH_DIR/terraform-providers/terraform-provider-google" "$TERRAFORM_COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "2.0.0"
+apply_patches "$PATCH_DIR/terraform-providers/$PROVIDER_NAME" "$TERRAFORM_COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "2.0.0"
 
 popd
 popd
 
-git clone magic-modules-branched/build/terraform ./terraform-generated
+git clone magic-modules-branched/build/$SHORT_NAME ./terraform-generated/$VERSION

.ci/magic-modules/generate-terraform.yml

@@ -19,3 +19,8 @@ outputs:
 
 run:
     path: magic-modules-branched/.ci/magic-modules/generate-terraform.sh
+
+params:
+  VERSION: ""
+  PROVIDER_NAME: ""
+  SHORT_NAME: ""

.ci/magic-modules/point-to-submodules.sh

@@ -44,11 +44,18 @@ for PRD in "${PRODUCT_ARRAY[@]}"; do
 done
 
 if [ "$TERRAFORM_ENABLED" = "true" ]; then
-  git config -f .gitmodules submodule.build/terraform.branch "$BRANCH"
-  git config -f .gitmodules submodule.build/terraform.url "[email protected]:$GH_USERNAME/terraform-provider-google.git"
-  git submodule sync build/terraform
-  ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/terraform"
-  git add build/terraform
+  IFS="," read -ra TERRAFORM_VERSIONS <<< "$TERRAFORM_VERSIONS"
+  for VERSION in "${TERRAFORM_VERSIONS[@]}"; do
+    IFS=":" read -ra TERRAFORM_DATA <<< "$VERSION"
+    PROVIDER_NAME="${TERRAFORM_DATA[0]}"
+    SUBMODULE_DIR="${TERRAFORM_DATA[1]}"
+
+    git config -f .gitmodules "submodule.build/$SUBMODULE_DIR.branch" "$BRANCH"
+    git config -f .gitmodules "submodule.build/$SUBMODULE_DIR.url" "[email protected]:$GH_USERNAME/$PROVIDER_NAME.git"
+    git submodule sync "build/$SUBMODULE_DIR"
+    ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/$SUBMODULE_DIR"
+    git add "build/$SUBMODULE_DIR"
+  done
 fi
 
 if [ "$ANSIBLE_ENABLED" = "true" ]; then

.ci/magic-modules/point-to-submodules.yml

@@ -23,6 +23,7 @@ params:
   GH_USERNAME: ""
   CREDS: ""
   TERRAFORM_ENABLED: false
+  TERRAFORM_VERSIONS: ""
   ANSIBLE_ENABLED: false
   INSPEC_ENABLED: false
   PUPPET_MODULES: ""

.ci/unit-tests/run.sh

@@ -9,8 +9,8 @@ set -x
 
 # Create GOPATH structure
 mkdir -p "${GOPATH}/src/github.com/terraform-providers"
-ln -s "${PWD}/$1" "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google"
+ln -s "${PWD}/magic-modules/build/$SHORT_NAME" "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 
-cd "${GOPATH}/src/github.com/terraform-providers/terraform-provider-google"
+cd "${GOPATH}/src/github.com/terraform-providers/$PROVIDER_NAME"
 
-go test -v ./google -parallel 16 -run '^Test' -timeout 1m
+go test -v ./$TEST_DIR -parallel 16 -run '^Test' -timeout 1m

.ci/unit-tests/task.yml

@@ -8,5 +8,7 @@ image_resource:
     tag: '1.9.3'
 run:
   path: magic-modules/.ci/unit-tests/run.sh
-  args:
-    - magic-modules/build/terraform/
+params:
+  PROVIDER_NAME: ""
+  SHORT_NAME: ""
+  TEST_DIR: ""

.ci/vars.tmpl

@@ -1,16 +1,35 @@
 {% set chef_modules = ['_bundle', 'auth', 'compute', 'sql', 'storage', 'spanner', 'container', 'dns', 'iam'] %}
 {% set puppet_modules = ['_bundle', 'auth', 'bigquery', 'compute', 'sql', 'storage', 'spanner', 'container', 'dns', 'pubsub', 'resourcemanager'] %}
 {% set puppet_no_release = ['bigquery'] %}
+{% set terraform_v = {
+    'ga': {
+      'provider_name': 'terraform-provider-google',
+      'short_name': 'terraform',
+      'test_dir': 'google'
+    },
+    'beta': {
+      'provider_name': 'terraform-provider-google-beta',
+      'short_name': 'terraform-beta',
+      'test_dir': 'google-beta'
+    }
+  }
+%}
 {% macro names_as_list(repo, names) -%}
 {% for name in names %}
 build/{{repo}}/{{name}} 
 {%- endfor %}
 {% endmacro -%}
+{% macro build_folder(names) -%}
+{% for name in names %}
+build/{{name}}
+{%- endfor %}
+{% endmacro -%}
 {% set puppet_submodules = names_as_list('puppet', puppet_modules).split() %}
 {% set chef_submodules = names_as_list('chef', chef_modules).split() %}
+{% set terraform_submodules = build_folder(terraform_v.values()|map(attribute='short_name')).split() %}
 {%
   set all_submodules = puppet_submodules + chef_submodules +
-     (['build/terraform'] + ['build/ansible'] + ['build/inspec'])
+     (terraform_submodules + ['build/ansible'] + ['build/inspec'])
 %}
 {% set all_submodules_yaml_format = '[' + ','.join(all_submodules) + ']' %}
 {% set chef_test_excludes = {
@@ -37,3 +56,9 @@ build/{{repo}}/{{name}}
     ]
   }
 %}
+{% macro serialize_terraform_properties(objs) -%}
+{% for obj in objs %}
+{{obj.provider_name}}:{{obj.short_name}}
+{%- endfor %}
+{% endmacro -%}
+{% set terraform_properties_serialized = serialize_terraform_properties(terraform_v.values()).split() %}

.github/PULL_REQUEST_TEMPLATE.md

@@ -21,6 +21,7 @@ this PR's changes from the commit messages for downstream commits.
 -----------------------------------------------------------------
 # [all]
 ## [terraform]
+### [terraform-beta]
 ## [puppet]
 ### [puppet-bigquery]
 ### [puppet-compute]

.gitmodules

@@ -81,6 +81,10 @@
 [submodule "build/chef/iam"]
 	path = build/chef/iam
 	url = [email protected]:GoogleCloudPlatform/chef-google-iam.git
+[submodule "build/terraform-beta"]
+	path = build/terraform-beta
+	url = [email protected]:terraform-providers/terraform-provider-google-beta.git
+	branch = 2.0.0
 [submodule "build/inspec"]
 	path = build/inspec
 	url = [email protected]:modular-magician/inspec-gcp.git

api/type.rb

@@ -310,6 +310,12 @@ def requires
         end
         [property_file]
       end
+
+      def exclude_if_not_in_version(version)
+        super
+        @item_type.exclude_if_not_in_version(version) \
+          if @item_type.is_a? NestedObject
+      end
     end
 
     # Represents an enum, and store is valid values
@@ -499,6 +505,11 @@ def all_properties
       def properties
         @properties.reject(&:exclude)
       end
+
+      def exclude_if_not_in_version(version)
+        super
+        @properties.each { |p| p.exclude_if_not_in_version(version) }
+      end
     end
 
     # Represents an array of name=value pairs, and stores its items' type