diff --git a/.ci/ci.yml.tmpl b/.ci/ci.yml.tmpl index 1631dd7320ee..f2d77b54f05f 100644 --- a/.ci/ci.yml.tmpl +++ b/.ci/ci.yml.tmpl @@ -64,30 +64,6 @@ resources: uri: git@github.com:((github-account.username))/inspec-gcp.git private_key: ((repo-key.private_key)) -{% for module in vars.puppet_modules %} - - name: puppet-{{module}}-intermediate - type: git-branch - source: -{% if module == '_bundle' %} - uri: git@github.com:((github-account.username))/puppet-google.git -{% else %} - uri: git@github.com:((github-account.username))/puppet-google-{{module}}.git -{% endif %} - private_key: ((repo-key.private_key)) -{% endfor %} - -{% for module in vars.chef_modules %} - - name: chef-{{module}}-intermediate - type: git-branch - source: -{% if module == '_bundle' %} - uri: git@github.com:((github-account.username))/chef-google.git -{% else %} - uri: git@github.com:((github-account.username))/chef-google-{{module}}.git -{% endif %} - private_key: ((repo-key.private_key)) -{% endfor %} - - name: mm-approved-prs type: github-pull-request source: @@ -181,43 +157,6 @@ jobs: branch_file: magic-modules-branched/branchname only_if_diff: true force: true - {% if vars.puppet_modules %} - - do: - # consumes: magic-modules-branched - # produces: puppet-generated - - task: generate-puppet - file: magic-modules-branched/.ci/magic-modules/generate-puppet.yml - params: - PRODUCTS: {{','.join(vars.puppet_modules)}} - - aggregate: - {% for module in vars.puppet_modules %} - - put: puppet-{{module}}-intermediate - params: - repository: puppet-generated/{{module}} - branch_file: magic-modules-branched/branchname - only_if_diff: true - force: true - {% endfor %} - {% endif %} - - {% if vars.chef_modules %} - - do: - # consumes: magic-modules-branched - # produces: chef-generated - - task: generate-chef - file: magic-modules-branched/.ci/magic-modules/generate-chef.yml - params: - PRODUCTS: {{','.join(vars.chef_modules)}} - - aggregate: - {% for module in vars.chef_modules %} - - put: chef-{{module}}-intermediate - params: - repository: chef-generated/{{module}} - branch_file: magic-modules-branched/branchname - only_if_diff: true - force: true - {% endfor %} - {% endif %} on_failure: put: magic-modules-new-prs params: @@ -233,8 +172,6 @@ jobs: # This needs to match the username for the 'intermediate' resources. GH_USERNAME: ((github-account.username)) CREDS: ((repo-key.private_key)) - PUPPET_MODULES: {{','.join(vars.puppet_modules)}} - CHEF_MODULES: {{','.join(vars.chef_modules)}} TERRAFORM_VERSIONS: "{{','.join(vars.terraform_properties_serialized)}}" TERRAFORM_ENABLED: true ANSIBLE_ENABLED: true @@ -317,70 +254,6 @@ jobs: context: inspec-tests path: magic-modules-new-prs - - name: puppet-test - plan: - - get: magic-modules - version: every - trigger: true - params: - submodules: - {% for module in vars.puppet_submodules %} - - {{module}} - {% endfor %} - passed: [mm-generate] - - aggregate: - {% for module in vars.puppet_modules %} - - task: test-{{module}} - file: magic-modules/.ci/unit-tests/puppet.yml - timeout: 30m - params: - PRODUCT: {{module}} - {%- if vars.puppet_test_excludes.get(module) %} - EXCLUDE_PATTERN: {{','.join(vars.puppet_test_excludes[module])}} - {%- endif %} - {% endfor %} - on_failure: - do: - - get: magic-modules-new-prs - passed: [mm-generate] - - put: magic-modules-new-prs - params: - status: failure - context: puppet-tests - path: magic-modules-new-prs - - - name: chef-test - plan: - - get: magic-modules - version: every - trigger: true - params: - submodules: - {% for module in vars.chef_submodules %} - - {{module}} - {% endfor %} - passed: [mm-generate] - - aggregate: - {% for module in vars.chef_modules %} - - task: test-{{module}} - file: magic-modules/.ci/unit-tests/chef.yml - timeout: 30m - params: - PRODUCT: {{module}} - {%- if vars.chef_test_excludes.get(module) %} - EXCLUDE_PATTERN: {{','.join(vars.chef_test_excludes[module])}} - {%- endif %} - {% endfor %} - on_failure: - do: - - get: magic-modules-new-prs - passed: [mm-generate] - - put: magic-modules-new-prs - params: - status: failure - context: chef-tests - path: magic-modules-new-prs - - name: create-prs plan: - get: magic-modules @@ -390,12 +263,6 @@ jobs: submodules: {{vars.all_submodules_yaml_format}} passed: - mm-generate - {%- if vars.chef_modules %} - - chef-test - {%- endif -%} - {%- if vars.puppet_modules %} - - puppet-test - {%- endif %} - terraform-test - ansible-test - inspec-test @@ -416,14 +283,6 @@ jobs: TERRAFORM_REPO_USER: terraform-providers ANSIBLE_REPO_USER: modular-magician INSPEC_REPO_USER: modular-magician - {%- if vars.puppet_modules %} - PUPPET_REPO_USER: GoogleCloudPlatform - PUPPET_MODULES: {{','.join(vars.puppet_modules)}} - {%- endif %} - {%- if vars.chef_modules %} - CHEF_REPO_USER: GoogleCloudPlatform - CHEF_MODULES: {{','.join(vars.chef_modules)}} - {%- endif %} TERRAFORM_VERSIONS: "{{','.join(vars.terraform_properties_serialized)}}" on_failure: put: magic-modules-new-prs @@ -466,23 +325,6 @@ jobs: # See comment on terraform-intermediate only_if_diff: true force: true - {% for module in vars.puppet_modules %} - - put: puppet-{{module}}-intermediate - params: - repository: magic-modules-with-comment/build/puppet/{{module}} - branch_file: magic-modules-with-comment/original_pr_branch_name - only_if_diff: true - force: true - {% endfor %} - - {% for module in vars.chef_modules %} - - put: chef-{{module}}-intermediate - params: - repository: magic-modules-with-comment/build/chef/{{module}} - branch_file: magic-modules-with-comment/original_pr_branch_name - only_if_diff: true - force: true - {% endfor %} # This isn't load-bearing - it's just aesthetic. It will also be a no-op the first # time through, it works the same way as the preceding push. diff --git a/.ci/magic-modules/create-pr.sh b/.ci/magic-modules/create-pr.sh index f1451ba18418..852fdb6c8fed 100755 --- a/.ci/magic-modules/create-pr.sh +++ b/.ci/magic-modules/create-pr.sh @@ -24,8 +24,6 @@ echo "$ORIGINAL_PR_BRANCH" > ./original_pr_branch_name # branch of the terraform submodule. All the submodules will be on the the same # branch name - we pick terraform because it's the first one the magician supported. BRANCH_NAME="$(git config -f .gitmodules --get submodule.build/terraform.branch)" -IFS="," read -ra PUPPET_PRODUCTS <<< "$PUPPET_MODULES" -IFS="," read -ra CHEF_PRODUCTS <<< "$CHEF_MODULES" IFS="," read -ra TERRAFORM_VERSIONS <<< "$TERRAFORM_VERSIONS" git checkout -b "$BRANCH_NAME" @@ -98,65 +96,15 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then popd fi - for PRD in "${PUPPET_PRODUCTS[@]}"; do - - pushd "build/puppet/$PRD" - - git log -1 --pretty=%B > ./downstream_body - echo "" >> ./downstream_body - echo "" >> ./downstream_body - if [ -n "$ORIGINAL_PR_USER" ]; then - echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body - fi - - git checkout -b "$BRANCH_NAME" - if [[ $PRD == *"_bundle"* ]]; then - repo="puppet-google" - else - repo="puppet-google-$PRD" - fi - if PUP_PR=$(hub pull-request -b "$PUPPET_REPO_USER/$repo:master" -F ./downstream_body); then - DEPENDENCIES="${DEPENDENCIES}depends: $PUP_PR ${NEWLINE}" - else - echo "Puppet $PRD - did not generate a PR." - fi - popd - done - - for PRD in "${CHEF_PRODUCTS[@]}"; do - - pushd "build/chef/$PRD" - - git log -1 --pretty=%B > ./downstream_body - echo "" >> ./downstream_body - echo "" >> ./downstream_body - if [ -n "$ORIGINAL_PR_USER" ]; then - echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body - fi - - git checkout -b "$BRANCH_NAME" - if [[ $PRD == *"_bundle"* ]]; then - repo="chef-google" - else - repo="chef-google-$PRD" - fi - if CHEF_PR=$(hub pull-request -b "$CHEF_REPO_USER/$repo:master" -F ./downstream_body); then - DEPENDENCIES="${DEPENDENCIES}depends: $CHEF_PR ${NEWLINE}" - else - echo "Chef $PRD - did not generate a PR." - fi - popd - done - if [ -z "$DEPENDENCIES" ]; then - cat << EOF > ./pr_comment + cat << EOF > ./pr_comment I am a robot that works on MagicModules PRs! I checked the downstream repositories (see README.md for which ones I can write to), and none of them seem to have any changes. Once this PR is approved, you can feel free to merge it without taking any further steps. EOF else - cat << EOF > ./pr_comment + cat << EOF > ./pr_comment I am a robot that works on MagicModules PRs! I built this PR into one or more PRs on other repositories, and when those are closed, this PR will also be merged and closed. @@ -181,16 +129,6 @@ else done fi - for PRD in "${PUPPET_PRODUCTS[@]}"; do - pushd "build/puppet/$PRD" - git branch -f "$ORIGINAL_PR_BRANCH" - popd - done - for PRD in "${CHEF_PRODUCTS[@]}"; do - pushd "build/chef/$PRD" - git branch -f "$ORIGINAL_PR_BRANCH" - popd - done # Note - we're interested in HEAD~1 here, not HEAD, because HEAD is the # generated code commit. :) cat << EOF > ./pr_comment diff --git a/.ci/magic-modules/create-pr.yml b/.ci/magic-modules/create-pr.yml index 8c5ffa611181..187c8c6aa266 100644 --- a/.ci/magic-modules/create-pr.yml +++ b/.ci/magic-modules/create-pr.yml @@ -25,8 +25,4 @@ params: TERRAFORM_REPO_USER: "" ANSIBLE_REPO_USER: "" INSPEC_REPO_USER: "" - PUPPET_REPO_USER: "" - PUPPET_MODULES: "" - CHEF_REPO_USER: "" - CHEF_MODULES: "" TERRAFORM_VERSIONS: "" diff --git a/.ci/magic-modules/generate-chef.yml b/.ci/magic-modules/generate-chef.yml deleted file mode 100644 index 1dd5382fa994..000000000000 --- a/.ci/magic-modules/generate-chef.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the CI repo. -# It spits out "chef-generated", a directory containing chef repos on a new branch -# (named after the HEAD commit on the PR), with the new generated code in it. -platform: linux - -image_resource: - type: docker-image - source: - repository: nmckinley/go-ruby-python - tag: '1.11-2.5-2.7' - -inputs: - - name: magic-modules-branched - -outputs: - - name: chef-generated - -run: - path: magic-modules-branched/.ci/magic-modules/generate-puppet-chef.sh - -params: - PRODUCTS: "" - PROVIDER: chef diff --git a/.ci/magic-modules/generate-puppet-chef.sh b/.ci/magic-modules/generate-puppet-chef.sh deleted file mode 100755 index 7bd2433b3264..000000000000 --- a/.ci/magic-modules/generate-puppet-chef.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules. -# It outputs "$PROVIDER-generated", a folder containing at least one non-submodule git repo containing the -# generated puppet/chef code. - -set -x -set -e -source "$(dirname "$0")/helpers.sh" -PATCH_DIR="$(pwd)/patches" - -IFS="," read -ra PRODUCT_ARRAY <<< "$PRODUCTS" -for PRD in "${PRODUCT_ARRAY[@]}"; do - pushd magic-modules-branched - LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)" - find build/"${PROVIDER}/${PRD}"/ -type f -not -name '.git*' -not -name '.last_run.json' -print0 | xargs -0 rm -rf -- - bundle install - # Running with the --debug flag will cause Concourse to crash - bundle exec compiler -p "products/$PRD" -e "$PROVIDER" -o "build/$PROVIDER/$PRD" - - # This command can crash - if that happens, the script should not fail. - set +e - COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag "$PROVIDER-$PRD" < .git/body)" - set -e - if [ -z "$COMMIT_MSG" ]; then - COMMIT_MSG="Magic Modules changes." - fi - - pushd "build/$PROVIDER/$PRD" - # These config entries will set the "committer". - git config --global user.email "magic-modules@google.com" - git config --global user.name "Modular Magician" - - git add -A - # Set the "author" to the commit's real author. - git commit -m "$COMMIT_MSG" --author="$LAST_COMMIT_AUTHOR" || true # don't crash if no changes - git checkout -B "$(cat ../../../branchname)" - apply_patches "$PATCH_DIR/GoogleCloudPlatform/$PROVIDER-google-$PRD" "$COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "master" - popd - popd - git clone "magic-modules-branched/build/$PROVIDER/$PRD" "$PROVIDER-generated/$PRD" - -done diff --git a/.ci/magic-modules/generate-puppet.yml b/.ci/magic-modules/generate-puppet.yml deleted file mode 100644 index 768a96f2eaad..000000000000 --- a/.ci/magic-modules/generate-puppet.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the patches. -# It spits out "puppet-generated", a directory containing puppet repos on a new branch -# (named after the HEAD commit on the PR), with the new generated code in it. -platform: linux - -image_resource: - type: docker-image - source: - repository: nmckinley/go-ruby-python - tag: '1.11-2.5-2.7' - -inputs: - - name: magic-modules-branched - - name: patches - -outputs: - - name: puppet-generated - -run: - path: magic-modules-branched/.ci/magic-modules/generate-puppet-chef.sh - -params: - PRODUCTS: "" - PROVIDER: puppet diff --git a/.ci/magic-modules/point-to-submodules.sh b/.ci/magic-modules/point-to-submodules.sh index b14dc85af643..0aa170095f8f 100755 --- a/.ci/magic-modules/point-to-submodules.sh +++ b/.ci/magic-modules/point-to-submodules.sh @@ -13,35 +13,6 @@ chmod 400 ~/github_private_key pushd magic-modules-branched BRANCH="$(cat ./branchname)" # Update this repo to track the submodules we just pushed: -IFS="," read -ra PRODUCT_ARRAY <<< "$PUPPET_MODULES" -for PRD in "${PRODUCT_ARRAY[@]}"; do - git config -f .gitmodules "submodule.build/puppet/$PRD.branch" "$BRANCH" - # Bundle repo does not use the same naming scheme as the others - if [[ $PRD == *"_bundle"* ]]; then - repo="puppet-google" - else - repo="puppet-google-$PRD" - fi - git config -f .gitmodules "submodule.build/puppet/$PRD.url" "git@github.com:$GH_USERNAME/$repo.git" - git submodule sync "build/puppet/$PRD" - ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/puppet/$PRD" - git add "build/puppet/$PRD" -done - -IFS="," read -ra PRODUCT_ARRAY <<< "$CHEF_MODULES" -for PRD in "${PRODUCT_ARRAY[@]}"; do - git config -f .gitmodules "submodule.build/chef/$PRD.branch" "$BRANCH" - # Bundle repo does not use the same naming scheme as the others - if [[ $PRD == *"_bundle"* ]]; then - repo="chef-google" - else - repo="chef-google-$PRD" - fi - git config -f .gitmodules "submodule.build/chef/$PRD.url" "git@github.com:$GH_USERNAME/$repo.git" - git submodule sync "build/chef/$PRD" - ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/chef/$PRD" - git add "build/chef/$PRD" -done if [ "$TERRAFORM_ENABLED" = "true" ]; then IFS="," read -ra TERRAFORM_VERSIONS <<< "$TERRAFORM_VERSIONS" diff --git a/.ci/magic-modules/point-to-submodules.yml b/.ci/magic-modules/point-to-submodules.yml index 2c423e8c5d4b..f76f46400fdb 100644 --- a/.ci/magic-modules/point-to-submodules.yml +++ b/.ci/magic-modules/point-to-submodules.yml @@ -26,6 +26,4 @@ params: TERRAFORM_VERSIONS: "" ANSIBLE_ENABLED: false INSPEC_ENABLED: false - PUPPET_MODULES: "" - CHEF_MODULES: "" diff --git a/.ci/unit-tests/chef.yml b/.ci/unit-tests/chef.yml deleted file mode 100644 index cc4b5dd9753e..000000000000 --- a/.ci/unit-tests/chef.yml +++ /dev/null @@ -1,14 +0,0 @@ -platform: linux -inputs: - - name: magic-modules -image_resource: - type: docker-image - source: - repository: ruby - tag: '2.5' -run: - path: magic-modules/.ci/unit-tests/puppet-chef.sh -params: - PRODUCT: "" - PROVIDER: chef - EXCLUDE_PATTERN: "" diff --git a/.ci/unit-tests/puppet-chef.sh b/.ci/unit-tests/puppet-chef.sh deleted file mode 100755 index 8e5a0d4002fa..000000000000 --- a/.ci/unit-tests/puppet-chef.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e -set -x - -pushd "magic-modules/build/$PROVIDER/$PRODUCT" -bundle install - -if [ $PROVIDER = "chef" ]; then - # TODO: https://github.com/GoogleCloudPlatform/magic-modules/issues/236 - # Re-enable chef tests by deleting this if block once the tests are fixed. - echo "Skipping tests... See issue #236" -elif [ -z "$EXCLUDE_PATTERN" ]; then - DISABLE_COVERAGE=true bundle exec parallel_rspec spec/ -else - # parallel_rspec doesn't support --exclude_pattern - IFS="," read -ra excluded <<< "$EXCLUDE_PATTERN" - filtered=$(find spec -name '*_spec.rb' $(printf "! -wholename %s " ${excluded[@]})) - DISABLE_COVERAGE=true bundle exec parallel_rspec ${filtered[@]} -fi - -popd diff --git a/.ci/unit-tests/puppet.yml b/.ci/unit-tests/puppet.yml deleted file mode 100644 index d628e72fb032..000000000000 --- a/.ci/unit-tests/puppet.yml +++ /dev/null @@ -1,14 +0,0 @@ -platform: linux -inputs: - - name: magic-modules -image_resource: - type: docker-image - source: - repository: ruby - tag: '2.5' -run: - path: magic-modules/.ci/unit-tests/puppet-chef.sh -params: - PRODUCT: "" - PROVIDER: puppet - EXCLUDE_PATTERN: "" diff --git a/.ci/vars.tmpl b/.ci/vars.tmpl index 5ad8ef11aef7..5ce55e16cb22 100644 --- a/.ci/vars.tmpl +++ b/.ci/vars.tmpl @@ -28,7 +28,7 @@ build/{{name}} {% set chef_submodules = names_as_list('chef', chef_modules).split() %} {% set terraform_submodules = build_folder(terraform_v.values()|map(attribute='short_name')).split() %} {% - set all_submodules = puppet_submodules + chef_submodules + + set all_submodules = (terraform_submodules + ['build/ansible'] + ['build/inspec']) %} {% set all_submodules_yaml_format = '[' + ','.join(all_submodules) + ']' %} @@ -42,20 +42,6 @@ build/{{name}} ] } %} -{% set puppet_test_excludes = { - 'compute': [ - 'spec/gcompute_instance_group_manager_provider_spec.rb', - 'spec/gcompute_instance_provider_spec.rb', - 'spec/gcompute_instance_template_provider_spec.rb', - 'spec/gcompute_target_https_proxy_provider_spec.rb', - 'spec/gcompute_target_ssl_proxy_provider_spec.rb', - 'spec/gcompute_vpn_tunnel_provider_spec.rb', - 'spec/puppetlint_spec.rb'], - 'bigquery': [ - 'spec/gbigquery_table_provider_spec.rb' - ] - } -%} {% macro serialize_terraform_properties(objs) -%} {% for obj in objs %} {{obj.provider_name}}:{{obj.short_name}} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 560f48c4d584..7f056f0ec94d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -22,23 +22,5 @@ this PR's changes from the commit messages for downstream commits. # [all] ## [terraform] ### [terraform-beta] -## [puppet] -### [puppet-bigquery] -### [puppet-compute] -### [puppet-container] -### [puppet-dns] -### [puppet-logging] -### [puppet-pubsub] -### [puppet-resourcemanager] -### [puppet-sql] -### [puppet-storage] -## [chef] -### [chef-compute] -### [chef-container] -### [chef-dns] -### [chef-logging] -### [chef-spanner] -### [chef-sql] -### [chef-storage] ## [ansible] ## [inspec] diff --git a/build/terraform b/build/terraform index 423ebf13bf9f..7474a1ec144f 160000 --- a/build/terraform +++ b/build/terraform @@ -1 +1 @@ -Subproject commit 423ebf13bf9feed9eb24278327401002b86c2855 +Subproject commit 7474a1ec144ff9b539840f1537b81c6b692b0de7