From 078067a62c8822852f5911cabf2a6c166dbf1beb Mon Sep 17 00:00:00 2001 From: Sam Levenick Date: Tue, 16 Oct 2018 15:42:34 -0700 Subject: [PATCH] Ground up (#557) * Add basic provider * Remove build * Remove extra variables * Add CI for inspec * Reference master branch for creating PRs * Remove bad comments * Add inspec to pr template * yml not yaml * Rubocop * Rubocopppp * Add submodule for inspec * Ci changes for inspec * Chmod inspec * master instead of devel * Add method to prevent auto generated tests for inspec * Add bash headers * Add rm for current repo * Revert "Add rm for current repo" This reverts commit de0dd27b7382c0c783aebe5c227a66aaa6ec3345. * Add inspec to rakefile --- .ci/ci.yml.tmpl | 49 ++++++++++++++++ .ci/magic-modules/create-pr.sh | 19 ++++++ .ci/magic-modules/create-pr.yml | 1 + .ci/magic-modules/generate-inspec.sh | 40 +++++++++++++ .ci/magic-modules/generate-inspec.yml | 21 +++++++ .ci/magic-modules/point-to-submodules.sh | 8 +++ .ci/magic-modules/point-to-submodules.yml | 1 + .ci/unit-tests/inspec.sh | 6 ++ .ci/unit-tests/inspec.yml | 14 +++++ .ci/vars.tmpl | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 1 + .gitmodules | 3 + Rakefile | 3 +- build/inspec | 1 + compiler.rb | 1 + products/compute/inspec.yaml | 29 +++++++++ provider/inspec.rb | 71 +++++++++++++++++++++++ provider/inspec/manifest.rb | 25 ++++++++ provider/inspec/property_override.rb | 36 ++++++++++++ provider/inspec/resource_override.rb | 53 +++++++++++++++++ templates/inspec/plural_resource.erb | 17 ++++++ templates/inspec/singular_resource.erb | 17 ++++++ 22 files changed, 416 insertions(+), 2 deletions(-) create mode 100755 .ci/magic-modules/generate-inspec.sh create mode 100644 .ci/magic-modules/generate-inspec.yml create mode 100755 .ci/unit-tests/inspec.sh create mode 100644 .ci/unit-tests/inspec.yml create mode 160000 build/inspec create mode 100644 products/compute/inspec.yaml create mode 100644 provider/inspec.rb create mode 100644 provider/inspec/manifest.rb create mode 100644 provider/inspec/property_override.rb create mode 100644 provider/inspec/resource_override.rb create mode 100644 templates/inspec/plural_resource.erb create mode 100644 templates/inspec/singular_resource.erb diff --git a/.ci/ci.yml.tmpl b/.ci/ci.yml.tmpl index e91635af26a0..d87f661869f9 100644 --- a/.ci/ci.yml.tmpl +++ b/.ci/ci.yml.tmpl @@ -56,6 +56,12 @@ resources: uri: git@github.com:((github-account.username))/ansible.git private_key: ((repo-key.private_key)) + - name: inspec-intermediate + type: git-branch + source: + uri: git@github.com:((github-account.username))/inspec-gcp.git + private_key: ((repo-key.private_key)) + {% for module in vars.puppet_modules %} - name: puppet-{{module}}-intermediate type: git-branch @@ -154,6 +160,18 @@ jobs: branch_file: magic-modules-branched/branchname only_if_diff: true force: true + - do: + # consumes: magic-modules-branched + # produces: inspec-generated + - task: generate-inspec + file: magic-modules-branched/.ci/magic-modules/generate-inspec.yml + # Puts 'inspec-generated' into the robot's fork. + - put: inspec-intermediate + params: + repository: inspec-generated + branch_file: magic-modules-branched/branchname + only_if_diff: true + force: true {% if vars.puppet_modules %} - do: # consumes: magic-modules-branched @@ -210,6 +228,7 @@ jobs: CHEF_MODULES: {{','.join(vars.chef_modules)}} TERRAFORM_ENABLED: true ANSIBLE_ENABLED: true + INSPEC_ENABLED: true # Push the magic modules branch that contains the updated submodules. - put: magic-modules @@ -260,6 +279,27 @@ jobs: context: ansible-tests path: magic-modules-new-prs + - name: inspec-test + plan: + - get: magic-modules + version: every + trigger: true + params: + submodules: [build/inspec] + passed: [mm-generate] + - task: test + file: magic-modules/.ci/unit-tests/inspec.yml + timeout: 30m + on_failure: + do: + - get: magic-modules-new-prs + passed: [mm-generate] + - put: magic-modules-new-prs + params: + status: failure + context: inspec-tests + path: magic-modules-new-prs + - name: puppet-test plan: - get: magic-modules @@ -341,6 +381,7 @@ jobs: {%- endif %} - terraform-test - ansible-test + - inspec-test - get: mm-initial-pr resource: magic-modules-new-prs passed: [mm-generate] @@ -357,6 +398,7 @@ jobs: # is what you change if you want to test this in a non-live environment. TERRAFORM_REPO_USER: terraform-providers ANSIBLE_REPO_USER: modular-magician + INSPEC_REPO_USER: modular-magician {%- if vars.puppet_modules %} PUPPET_REPO_USER: GoogleCloudPlatform PUPPET_MODULES: {{','.join(vars.puppet_modules)}} @@ -397,6 +439,13 @@ jobs: # See comment on terraform-intermediate only_if_diff: true force: true + - put: inspec-intermediate + params: + repository: magic-modules-with-comment/build/inspec + branch_file: magic-modules-with-comment/original_pr_branch_name + # See comment on terraform-intermediate + only_if_diff: true + force: true {% for module in vars.puppet_modules %} - put: puppet-{{module}}-intermediate params: diff --git a/.ci/magic-modules/create-pr.sh b/.ci/magic-modules/create-pr.sh index 5365ffce205f..041f98dcb2a2 100755 --- a/.ci/magic-modules/create-pr.sh +++ b/.ci/magic-modules/create-pr.sh @@ -71,6 +71,25 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then popd fi + if [ -n "$INSPEC_REPO_USER" ]; then + pushd build/inspec + + git log -1 --pretty=%B > ./downstream_body + echo "" >> ./downstream_body + echo "" >> ./downstream_body + if [ -n "$ORIGINAL_PR_USER" ]; then + echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body + fi + + git checkout -b "$BRANCH_NAME" + if INSPEC_PR=$(hub pull-request -b "$INSPEC_REPO_USER/inspec:master" -F ./downstream_body); then + DEPENDENCIES="${DEPENDENCIES}depends: $INSPEC_PR ${NEWLINE}" + else + echo "InSpec - did not generate a PR." + fi + popd + fi + for PRD in "${PUPPET_PRODUCTS[@]}"; do pushd "build/puppet/$PRD" diff --git a/.ci/magic-modules/create-pr.yml b/.ci/magic-modules/create-pr.yml index 859d1a049a91..735318704f06 100644 --- a/.ci/magic-modules/create-pr.yml +++ b/.ci/magic-modules/create-pr.yml @@ -24,6 +24,7 @@ params: GITHUB_TOKEN: "" TERRAFORM_REPO_USER: "" ANSIBLE_REPO_USER: "" + INSPEC_REPO_USER: "" PUPPET_REPO_USER: "" PUPPET_MODULES: "" CHEF_REPO_USER: "" diff --git a/.ci/magic-modules/generate-inspec.sh b/.ci/magic-modules/generate-inspec.sh new file mode 100755 index 000000000000..0896aadb1557 --- /dev/null +++ b/.ci/magic-modules/generate-inspec.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules. +# It outputs "inspec-generated", a non-submodule git repo containing the generated inspec code. + +set -x +set -e +source "$(dirname "$0")/helpers.sh" +PATCH_DIR="$(pwd)/patches" +pushd magic-modules-branched +LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)" +bundle install +for i in $(find products/ -name 'inspec.yaml' -printf '%h\n'); +do + bundle exec compiler -p $i -e inspec -o "build/inspec/" +done + +# This command can crash - if that happens, the script should not fail. +set +e +INSPEC_COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag inspec < .git/body)" +set -e +if [ -z "$INSPEC_COMMIT_MSG" ]; then + INSPEC_COMMIT_MSG="Magic Modules changes." +fi + +pushd "build/inspec" +# These config entries will set the "committer". +git config --global user.email "magic-modules@google.com" +git config --global user.name "Modular Magician" + +git add -A +# Set the "author" to the commit's real author. +git commit -m "$INSPEC_COMMIT_MSG" --author="$LAST_COMMIT_AUTHOR" || true # don't crash if no changes +git checkout -B "$(cat ../../branchname)" + +apply_patches "$PATCH_DIR/modular-magician/inspec-gcp" "$INSPEC_COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "master" +popd +popd + +git clone magic-modules-branched/build/inspec ./inspec-generated diff --git a/.ci/magic-modules/generate-inspec.yml b/.ci/magic-modules/generate-inspec.yml new file mode 100644 index 000000000000..575cd28f6d2f --- /dev/null +++ b/.ci/magic-modules/generate-inspec.yml @@ -0,0 +1,21 @@ +--- +# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the patches. +# It spits out "inspec-generated", an inspec repo on a new branch (named after the +# HEAD commit on the PR), with the new generated code in it. +platform: linux + +image_resource: + type: docker-image + source: + repository: nmckinley/go-ruby-python + tag: '1.11-2.5-2.7' + +inputs: + - name: magic-modules-branched + - name: patches + +outputs: + - name: inspec-generated + +run: + path: magic-modules-branched/.ci/magic-modules/generate-inspec.sh diff --git a/.ci/magic-modules/point-to-submodules.sh b/.ci/magic-modules/point-to-submodules.sh index e852cdb96506..39ba92110781 100755 --- a/.ci/magic-modules/point-to-submodules.sh +++ b/.ci/magic-modules/point-to-submodules.sh @@ -59,6 +59,14 @@ if [ "$ANSIBLE_ENABLED" = "true" ]; then git add build/ansible fi +if [ "$INSPEC_ENABLED" = "true" ]; then + git config -f .gitmodules submodule.build/inspec.branch "$BRANCH" + git config -f .gitmodules submodule.build/inspec.url "git@github.com:$GH_USERNAME/inspec-gcp.git" + git submodule sync build/inspec + ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/inspec" + git add build/inspec +fi + # Commit those changes so that they can be tested in the next phase. git add .gitmodules git config --global user.email "magic-modules@google.com" diff --git a/.ci/magic-modules/point-to-submodules.yml b/.ci/magic-modules/point-to-submodules.yml index 82f5c5653674..f91ff55283b8 100644 --- a/.ci/magic-modules/point-to-submodules.yml +++ b/.ci/magic-modules/point-to-submodules.yml @@ -24,6 +24,7 @@ params: CREDS: "" TERRAFORM_ENABLED: false ANSIBLE_ENABLED: false + INSPEC_ENABLED: false PUPPET_MODULES: "" CHEF_MODULES: "" diff --git a/.ci/unit-tests/inspec.sh b/.ci/unit-tests/inspec.sh new file mode 100755 index 000000000000..eb1427683a1f --- /dev/null +++ b/.ci/unit-tests/inspec.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e +set -x + +echo 'TODO slevenick write tests' \ No newline at end of file diff --git a/.ci/unit-tests/inspec.yml b/.ci/unit-tests/inspec.yml new file mode 100644 index 000000000000..d189d153caa6 --- /dev/null +++ b/.ci/unit-tests/inspec.yml @@ -0,0 +1,14 @@ +platform: linux +inputs: + - name: magic-modules +image_resource: + type: docker-image + source: + repository: ruby + tag: '2.5' +run: + path: magic-modules/.ci/unit-tests/inspec.sh +params: + PRODUCT: "" + PROVIDER: inspec + EXCLUDE_PATTERN: "" diff --git a/.ci/vars.tmpl b/.ci/vars.tmpl index aac853d7cdd3..f5aaa68e4548 100644 --- a/.ci/vars.tmpl +++ b/.ci/vars.tmpl @@ -10,7 +10,7 @@ build/{{repo}}/{{name}} {% set chef_submodules = names_as_list('chef', chef_modules).split() %} {% set all_submodules = puppet_submodules + chef_submodules + - (['build/terraform'] + ['build/ansible']) + (['build/terraform'] + ['build/ansible'] + ['build/inspec']) %} {% set all_submodules_yaml_format = '[' + ','.join(all_submodules) + ']' %} {% set chef_test_excludes = { diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index be8c9d4a27dc..bfab3ccbe1b1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -40,3 +40,4 @@ this PR's changes from the commit messages for downstream commits. ### [chef-sql] ### [chef-storage] ## [ansible] +## [inspec] diff --git a/.gitmodules b/.gitmodules index 641426918029..a0c2eed20449 100644 --- a/.gitmodules +++ b/.gitmodules @@ -81,3 +81,6 @@ [submodule "build/chef/iam"] path = build/chef/iam url = git@github.com:GoogleCloudPlatform/chef-google-iam.git +[submodule "build/inspec"] + path = build/inspec + url = git@github.com:modular-magician/inspec-gcp.git diff --git a/Rakefile b/Rakefile index da3fce97ec10..4625a1e33a43 100644 --- a/Rakefile +++ b/Rakefile @@ -19,7 +19,8 @@ PROVIDER_FOLDERS = { ansible: 'build/ansible', puppet: 'build/puppet/%s', chef: 'build/chef/%s', - terraform: 'build/terraform' + terraform: 'build/terraform', + inspec: 'build/inspec' }.freeze # Requires diff --git a/build/inspec b/build/inspec new file mode 160000 index 000000000000..faf47ee4071d --- /dev/null +++ b/build/inspec @@ -0,0 +1 @@ +Subproject commit faf47ee4071d8c4c4bc7ca62cb904ffa0fbd1271 diff --git a/compiler.rb b/compiler.rb index 97dddcbf738a..68249439e524 100755 --- a/compiler.rb +++ b/compiler.rb @@ -31,6 +31,7 @@ require 'provider/chef' require 'provider/chef/bundle' require 'provider/example' +require 'provider/inspec' require 'provider/puppet' require 'provider/puppet/bundle' require 'provider/terraform' diff --git a/products/compute/inspec.yaml b/products/compute/inspec.yaml new file mode 100644 index 000000000000..6523b93ef4ab --- /dev/null +++ b/products/compute/inspec.yaml @@ -0,0 +1,29 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Provider::Inspec::Config +manifest: !ruby/object:Provider::Inspec::Manifest + version: '0.1.0' + source: 'FIXME' + issues: 'FIXME' + summary: 'InSpec resources for verifying GCP infrastructure' + description: | + InSpec resources for verifying GCP infrastructure +files: !ruby/object:Provider::Config::Files +style: +functions: +changelog: + - !ruby/object:Provider::Config::Changelog + version: '0.1.0' + date: 2017-10-04T10:00:00-0700 + general: 'Initial release' diff --git a/provider/inspec.rb b/provider/inspec.rb new file mode 100644 index 000000000000..7f66da907f30 --- /dev/null +++ b/provider/inspec.rb @@ -0,0 +1,71 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'provider/config' +require 'provider/core' +require 'provider/inspec/manifest' +require 'provider/inspec/resource_override' +require 'provider/inspec/property_override' + +module Provider + # Code generator for Example Cookbooks that manage Google Cloud Platform + # resources. + class Inspec < Provider::Core + # Settings for the provider + class Config < Provider::Config + attr_reader :manifest + def provider + Provider::Inspec + end + + def resource_override + Provider::Inspec::ResourceOverride + end + + def property_override + Provider::Inspec::PropertyOverride + end + end + + # This function uses the resource templates to create singular and plural + # resources that can be used by InSpec + def generate_resource(data) + target_folder = File.join(data[:output_folder], 'inspec') + FileUtils.mkpath target_folder + name = data[:object].name.underscore + generate_resource_file data.clone.merge( + default_template: 'templates/inspec/singular_resource.erb', + out_file: File.join(target_folder, "google_#{data[:product_name]}_#{name}.rb") + ) + generate_resource_file data.clone.merge( + default_template: 'templates/inspec/plural_resource.erb', + out_file: File.join(target_folder, "google_#{data[:product_name]}_#{name}s.rb") + ) + end + + # TODO? + def generate_resource_tests(data) end + + def generate_base_property(data) end + + def generate_simple_property(type, data) end + + def generate_typed_array(data, prop) end + + def emit_resourceref_object(data) end + + def emit_nested_object(data) end + + def generate_network_datas(data, object) end + end +end diff --git a/provider/inspec/manifest.rb b/provider/inspec/manifest.rb new file mode 100644 index 000000000000..e2502f5adf76 --- /dev/null +++ b/provider/inspec/manifest.rb @@ -0,0 +1,25 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'provider/core' + +module Provider + class Inspec < Provider::Core + # Metadata for manifest.json + class Manifest < Api::Object + def validate + super + end + end + end +end diff --git a/provider/inspec/property_override.rb b/provider/inspec/property_override.rb new file mode 100644 index 000000000000..67d3deabf746 --- /dev/null +++ b/provider/inspec/property_override.rb @@ -0,0 +1,36 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'api/object' +require 'provider/abstract_core' +require 'provider/property_override' + +module Provider + class Inspec < Provider::Core + # Collection of fields allowed in the PropertyOverride section for + # inspec. All fields should be `attr_reader :` + module OverrideFields + end + + # inspec-specific overrides to api.yaml. + class PropertyOverride < Provider::PropertyOverride + include OverrideFields + + private + + def overriden + Provider::Inspec::OverrideFields + end + end + end +end diff --git a/provider/inspec/resource_override.rb b/provider/inspec/resource_override.rb new file mode 100644 index 000000000000..b7771b20d53b --- /dev/null +++ b/provider/inspec/resource_override.rb @@ -0,0 +1,53 @@ +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'provider/core' +require 'provider/resource_override' + +module Provider + class Inspec < Provider::Core + # inspec specific properties to be added to Api::Resource + module OverrideProperties + attr_reader :manual + end + + # Custom inspec code to handle type convergence operations + class Handlers < Api::Object + def validate + super + end + end + + # Product specific overriden properties for inspec + class ResourceOverride < Provider::ResourceOverride + include OverrideProperties + + def validate + assign_defaults + + super + check_property :manual, :boolean + end + + private + + def assign_defaults + default_value_property :manual, false + end + + def overriden + Provider::Inspec::OverrideProperties + end + end + end +end diff --git a/templates/inspec/plural_resource.erb b/templates/inspec/plural_resource.erb new file mode 100644 index 000000000000..3628efec92c1 --- /dev/null +++ b/templates/inspec/plural_resource.erb @@ -0,0 +1,17 @@ +<%# The license inside this block applies to this file. +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :ruby) -%> \ No newline at end of file diff --git a/templates/inspec/singular_resource.erb b/templates/inspec/singular_resource.erb new file mode 100644 index 000000000000..3628efec92c1 --- /dev/null +++ b/templates/inspec/singular_resource.erb @@ -0,0 +1,17 @@ +<%# The license inside this block applies to this file. +# Copyright 2017 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +-%> +<%= compile 'templates/license.erb' -%> + +<%= lines(autogen_notice :ruby) -%> \ No newline at end of file