diff --git a/.ci/ci.yml.tmpl b/.ci/ci.yml.tmpl index bd2ceb69ec9c..1119c8a62c3c 100644 --- a/.ci/ci.yml.tmpl +++ b/.ci/ci.yml.tmpl @@ -56,6 +56,12 @@ resources: uri: git@github.com:((github-account.username))/ansible.git private_key: ((repo-key.private_key)) + - name: inspec-intermediate + type: git-branch + source: + uri: git@github.com:((github-account.username))/inspec-gcp.git + private_key: ((repo-key.private_key)) + {% for module in vars.puppet_modules %} - name: puppet-{{module}}-intermediate type: git-branch @@ -154,6 +160,18 @@ jobs: branch_file: magic-modules-branched/branchname only_if_diff: true force: true + - do: + # consumes: magic-modules-branched + # produces: inspec-generated + - task: generate-inspec + file: magic-modules-branched/.ci/magic-modules/generate-inspec.yml + # Puts 'inspec-generated' into the robot's fork. + - put: inspec-intermediate + params: + repository: inspec-generated + branch_file: magic-modules-branched/branchname + only_if_diff: true + force: true {% if vars.puppet_modules %} - do: # consumes: magic-modules-branched @@ -210,6 +228,7 @@ jobs: CHEF_MODULES: {{','.join(vars.chef_modules)}} TERRAFORM_ENABLED: true ANSIBLE_ENABLED: true + INSPEC_ENABLED: true # Push the magic modules branch that contains the updated submodules. - put: magic-modules @@ -260,6 +279,27 @@ jobs: context: ansible-tests path: magic-modules-new-prs + - name: inspec-test + plan: + - get: magic-modules + version: every + trigger: true + params: + submodules: [build/inspec] + passed: [mm-generate] + - task: test + file: magic-modules/.ci/unit-tests/inspec.yml + timeout: 30m + on_failure: + do: + - get: magic-modules-new-prs + passed: [mm-generate] + - put: magic-modules-new-prs + params: + status: failure + context: inspec-tests + path: magic-modules-new-prs + - name: puppet-test plan: - get: magic-modules @@ -341,6 +381,7 @@ jobs: {%- endif %} - terraform-test - ansible-test + - inspec-test - get: mm-initial-pr resource: magic-modules-new-prs passed: [mm-generate] @@ -357,6 +398,7 @@ jobs: # is what you change if you want to test this in a non-live environment. TERRAFORM_REPO_USER: terraform-providers ANSIBLE_REPO_USER: modular-magician + INSPEC_REPO_USER: modular-magician {%- if vars.puppet_modules %} PUPPET_REPO_USER: GoogleCloudPlatform PUPPET_MODULES: {{','.join(vars.puppet_modules)}} @@ -397,6 +439,13 @@ jobs: # See comment on terraform-intermediate only_if_diff: true force: true + - put: inspec-intermediate + params: + repository: magic-modules-with-comment/build/inspec + branch_file: magic-modules-with-comment/original_pr_branch_name + # See comment on terraform-intermediate + only_if_diff: true + force: true {% for module in vars.puppet_modules %} - put: puppet-{{module}}-intermediate params: diff --git a/.ci/magic-modules/create-pr.sh b/.ci/magic-modules/create-pr.sh index 5365ffce205f..f4f1a3107c20 100755 --- a/.ci/magic-modules/create-pr.sh +++ b/.ci/magic-modules/create-pr.sh @@ -71,6 +71,25 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then popd fi + if [ -n "$INSPEC_REPO_USER" ]; then + pushd build/inspec + + git log -1 --pretty=%B > ./downstream_body + echo "" >> ./downstream_body + echo "" >> ./downstream_body + if [ -n "$ORIGINAL_PR_USER" ]; then + echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body + fi + + git checkout -b "$BRANCH_NAME" + if INSPEC_PR=$(hub pull-request -b "$INSPEC_REPO_USER/inspec:devel" -F ./downstream_body); then + DEPENDENCIES="${DEPENDENCIES}depends: $INSPEC_PR ${NEWLINE}" + else + echo "InSpec - did not generate a PR." + fi + popd + fi + for PRD in "${PUPPET_PRODUCTS[@]}"; do pushd "build/puppet/$PRD" diff --git a/.ci/magic-modules/create-pr.yml b/.ci/magic-modules/create-pr.yml index 859d1a049a91..735318704f06 100644 --- a/.ci/magic-modules/create-pr.yml +++ b/.ci/magic-modules/create-pr.yml @@ -24,6 +24,7 @@ params: GITHUB_TOKEN: "" TERRAFORM_REPO_USER: "" ANSIBLE_REPO_USER: "" + INSPEC_REPO_USER: "" PUPPET_REPO_USER: "" PUPPET_MODULES: "" CHEF_REPO_USER: "" diff --git a/.ci/magic-modules/generate-inspec.sh b/.ci/magic-modules/generate-inspec.sh new file mode 100644 index 000000000000..ac9f921c8f4e --- /dev/null +++ b/.ci/magic-modules/generate-inspec.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules. +# It outputs "inspec-generated", a non-submodule git repo containing the generated inspec code. + +set -x +set -e +source "$(dirname "$0")/helpers.sh" +PATCH_DIR="$(pwd)/patches" +pushd magic-modules-branched +LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)" +bundle install +for i in $(find products/ -name 'inspec.yaml' -printf '%h\n'); +do + bundle exec compiler -p $i -e inspec -o "build/inspec/" +done + +# This command can crash - if that happens, the script should not fail. +set +e +INSPEC_COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag inspec < .git/body)" +set -e +if [ -z "$INSPEC_COMMIT_MSG" ]; then + INSPEC_COMMIT_MSG="Magic Modules changes." +fi + +pushd "build/inspec" +# These config entries will set the "committer". +git config --global user.email "magic-modules@google.com" +git config --global user.name "Modular Magician" + +git add -A +# Set the "author" to the commit's real author. +git commit -m "$INSPEC_COMMIT_MSG" --author="$LAST_COMMIT_AUTHOR" || true # don't crash if no changes +git checkout -B "$(cat ../../branchname)" + +apply_patches "$PATCH_DIR/modular-magician/inspec" "$INSPEC_COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "devel" +popd +popd + +git clone magic-modules-branched/build/inspec ./inspec-generated diff --git a/.ci/magic-modules/generate-inspec.yaml b/.ci/magic-modules/generate-inspec.yaml new file mode 100644 index 000000000000..575cd28f6d2f --- /dev/null +++ b/.ci/magic-modules/generate-inspec.yaml @@ -0,0 +1,21 @@ +--- +# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the patches. +# It spits out "inspec-generated", an inspec repo on a new branch (named after the +# HEAD commit on the PR), with the new generated code in it. +platform: linux + +image_resource: + type: docker-image + source: + repository: nmckinley/go-ruby-python + tag: '1.11-2.5-2.7' + +inputs: + - name: magic-modules-branched + - name: patches + +outputs: + - name: inspec-generated + +run: + path: magic-modules-branched/.ci/magic-modules/generate-inspec.sh diff --git a/.ci/magic-modules/point-to-submodules.sh b/.ci/magic-modules/point-to-submodules.sh index e852cdb96506..39ba92110781 100755 --- a/.ci/magic-modules/point-to-submodules.sh +++ b/.ci/magic-modules/point-to-submodules.sh @@ -59,6 +59,14 @@ if [ "$ANSIBLE_ENABLED" = "true" ]; then git add build/ansible fi +if [ "$INSPEC_ENABLED" = "true" ]; then + git config -f .gitmodules submodule.build/inspec.branch "$BRANCH" + git config -f .gitmodules submodule.build/inspec.url "git@github.com:$GH_USERNAME/inspec-gcp.git" + git submodule sync build/inspec + ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/inspec" + git add build/inspec +fi + # Commit those changes so that they can be tested in the next phase. git add .gitmodules git config --global user.email "magic-modules@google.com" diff --git a/.ci/magic-modules/point-to-submodules.yml b/.ci/magic-modules/point-to-submodules.yml index 82f5c5653674..f91ff55283b8 100644 --- a/.ci/magic-modules/point-to-submodules.yml +++ b/.ci/magic-modules/point-to-submodules.yml @@ -24,6 +24,7 @@ params: CREDS: "" TERRAFORM_ENABLED: false ANSIBLE_ENABLED: false + INSPEC_ENABLED: false PUPPET_MODULES: "" CHEF_MODULES: "" diff --git a/.ci/unit-tests/inspec.sh b/.ci/unit-tests/inspec.sh new file mode 100644 index 000000000000..84b244dce666 --- /dev/null +++ b/.ci/unit-tests/inspec.sh @@ -0,0 +1 @@ +echo 'Testing!' \ No newline at end of file diff --git a/.ci/unit-tests/inspec.yaml b/.ci/unit-tests/inspec.yaml new file mode 100644 index 000000000000..3b946c214ece --- /dev/null +++ b/.ci/unit-tests/inspec.yaml @@ -0,0 +1,14 @@ +platform: linux +inputs: + - name: magic-modules +image_resource: + type: docker-image + source: + repository: ruby + tag: '2.5' +run: + path: magic-modules/.ci/unit-tests/inspec.sh +params: + PRODUCT: "" + PROVIDER: chef + EXCLUDE_PATTERN: "" diff --git a/.ci/vars.tmpl b/.ci/vars.tmpl index aac853d7cdd3..f5aaa68e4548 100644 --- a/.ci/vars.tmpl +++ b/.ci/vars.tmpl @@ -10,7 +10,7 @@ build/{{repo}}/{{name}} {% set chef_submodules = names_as_list('chef', chef_modules).split() %} {% set all_submodules = puppet_submodules + chef_submodules + - (['build/terraform'] + ['build/ansible']) + (['build/terraform'] + ['build/ansible'] + ['build/inspec']) %} {% set all_submodules_yaml_format = '[' + ','.join(all_submodules) + ']' %} {% set chef_test_excludes = {