Ground up (#557)

* Add basic provider

* Remove build

* Remove extra variables

* Add CI for inspec

* Reference master branch for creating PRs

* Remove bad comments

* Add inspec to pr template

* yml not yaml

* Rubocop

* Rubocopppp

* Add submodule for inspec

* Ci changes for inspec

* Chmod inspec

* master instead of devel

* Add method to prevent auto generated tests for inspec

* Add bash headers

* Add rm for current repo

* Revert "Add rm for current repo"

This reverts commit de0dd27.

* Add inspec to rakefile

.ci/ci.yml.tmpl

@@ -56,6 +56,12 @@ resources:
           uri: [email protected]:((github-account.username))/ansible.git
           private_key: ((repo-key.private_key))
 
+    - name: inspec-intermediate
+      type: git-branch
+      source:
+          uri: [email protected]:((github-account.username))/inspec-gcp.git
+          private_key: ((repo-key.private_key))
+
 {% for module in vars.puppet_modules %}
     - name: puppet-{{module}}-intermediate
       type: git-branch
@@ -154,6 +160,18 @@ jobs:
                     branch_file: magic-modules-branched/branchname
                     only_if_diff: true
                     force: true
+            - do:
+                # consumes: magic-modules-branched
+                # produces: inspec-generated
+              - task: generate-inspec
+                file: magic-modules-branched/.ci/magic-modules/generate-inspec.yml
+              # Puts 'inspec-generated' into the robot's fork.
+              - put: inspec-intermediate
+                params:
+                    repository: inspec-generated
+                    branch_file: magic-modules-branched/branchname
+                    only_if_diff: true
+                    force: true
           {% if vars.puppet_modules %}
             - do:
                 # consumes: magic-modules-branched
@@ -210,6 +228,7 @@ jobs:
                 CHEF_MODULES: {{','.join(vars.chef_modules)}}
                 TERRAFORM_ENABLED: true
                 ANSIBLE_ENABLED: true
+                INSPEC_ENABLED: true
 
           # Push the magic modules branch that contains the updated submodules.
           - put: magic-modules
@@ -260,6 +279,27 @@ jobs:
                   context: ansible-tests
                   path: magic-modules-new-prs
 
+    - name: inspec-test
+      plan:
+        - get: magic-modules
+          version: every
+          trigger: true
+          params:
+              submodules: [build/inspec]
+          passed: [mm-generate]
+        - task: test
+          file: magic-modules/.ci/unit-tests/inspec.yml
+          timeout: 30m
+          on_failure:
+              do:
+              - get: magic-modules-new-prs
+                passed: [mm-generate]
+              - put: magic-modules-new-prs
+                params:
+                  status: failure
+                  context: inspec-tests
+                  path: magic-modules-new-prs
+
     - name: puppet-test
       plan:
         - get: magic-modules
@@ -341,6 +381,7 @@ jobs:
               {%- endif %}
               - terraform-test
               - ansible-test
+              - inspec-test
           - get: mm-initial-pr
             resource: magic-modules-new-prs
             passed: [mm-generate]
@@ -357,6 +398,7 @@ jobs:
                 # is what you change if you want to test this in a non-live environment.
                 TERRAFORM_REPO_USER: terraform-providers
                 ANSIBLE_REPO_USER: modular-magician
+                INSPEC_REPO_USER: modular-magician
                 {%- if vars.puppet_modules %}
                 PUPPET_REPO_USER: GoogleCloudPlatform
                 PUPPET_MODULES: {{','.join(vars.puppet_modules)}}
@@ -397,6 +439,13 @@ jobs:
                 # See comment on terraform-intermediate
                 only_if_diff: true
                 force: true
+            - put: inspec-intermediate
+              params:
+                repository: magic-modules-with-comment/build/inspec
+                branch_file: magic-modules-with-comment/original_pr_branch_name
+                # See comment on terraform-intermediate
+                only_if_diff: true
+                force: true
           {% for module in vars.puppet_modules %}
             - put: puppet-{{module}}-intermediate
               params:

.ci/magic-modules/create-pr.sh

@@ -71,6 +71,25 @@ if [ "$BRANCH_NAME" = "$ORIGINAL_PR_BRANCH" ]; then
     popd
   fi
 
+    if [ -n "$INSPEC_REPO_USER" ]; then
+    pushd build/inspec
+
+    git log -1 --pretty=%B > ./downstream_body
+    echo "" >> ./downstream_body
+    echo "<!-- This change is generated by MagicModules. -->" >> ./downstream_body
+    if [ -n "$ORIGINAL_PR_USER" ]; then
+      echo "/cc @$ORIGINAL_PR_USER" >> ./downstream_body
+    fi
+
+    git checkout -b "$BRANCH_NAME"
+    if INSPEC_PR=$(hub pull-request -b "$INSPEC_REPO_USER/inspec:master" -F ./downstream_body); then
+      DEPENDENCIES="${DEPENDENCIES}depends: $INSPEC_PR ${NEWLINE}"
+    else
+      echo "InSpec - did not generate a PR."
+    fi
+    popd
+  fi
+
   for PRD in "${PUPPET_PRODUCTS[@]}"; do
 
     pushd "build/puppet/$PRD"

.ci/magic-modules/create-pr.yml

@@ -24,6 +24,7 @@ params:
   GITHUB_TOKEN: ""
   TERRAFORM_REPO_USER: ""
   ANSIBLE_REPO_USER: ""
+  INSPEC_REPO_USER: ""
   PUPPET_REPO_USER: ""
   PUPPET_MODULES: ""
   CHEF_REPO_USER: ""

.ci/magic-modules/generate-inspec.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+# This script takes in 'magic-modules-branched', a git repo tracking the head of a PR against magic-modules.
+# It outputs "inspec-generated", a non-submodule git repo containing the generated inspec code.
+
+set -x
+set -e
+source "$(dirname "$0")/helpers.sh"
+PATCH_DIR="$(pwd)/patches"
+pushd magic-modules-branched
+LAST_COMMIT_AUTHOR="$(git log --pretty="%an <%ae>" -n1 HEAD)"
+bundle install
+for i in $(find products/ -name 'inspec.yaml' -printf '%h\n');
+do
+  bundle exec compiler -p $i -e inspec -o "build/inspec/"
+done
+
+# This command can crash - if that happens, the script should not fail.
+set +e
+INSPEC_COMMIT_MSG="$(python .ci/magic-modules/extract_from_pr_description.py --tag inspec < .git/body)"
+set -e
+if [ -z "$INSPEC_COMMIT_MSG" ]; then
+  INSPEC_COMMIT_MSG="Magic Modules changes."
+fi
+
+pushd "build/inspec"
+# These config entries will set the "committer".
+git config --global user.email "[email protected]"
+git config --global user.name "Modular Magician"
+
+git add -A
+# Set the "author" to the commit's real author.
+git commit -m "$INSPEC_COMMIT_MSG" --author="$LAST_COMMIT_AUTHOR" || true  # don't crash if no changes
+git checkout -B "$(cat ../../branchname)"
+
+apply_patches "$PATCH_DIR/modular-magician/inspec-gcp" "$INSPEC_COMMIT_MSG" "$LAST_COMMIT_AUTHOR" "master"
+popd
+popd
+
+git clone magic-modules-branched/build/inspec ./inspec-generated

.ci/magic-modules/generate-inspec.yml

@@ -0,0 +1,21 @@
+---
+# This file takes two inputs: magic-modules-branched in detached-HEAD state, and the patches.
+# It spits out "inspec-generated", an inspec repo on a new branch (named after the
+# HEAD commit on the PR), with the new generated code in it.
+platform: linux
+
+image_resource:
+    type: docker-image
+    source:
+        repository: nmckinley/go-ruby-python
+        tag: '1.11-2.5-2.7'
+
+inputs:
+    - name: magic-modules-branched
+    - name: patches
+
+outputs:
+    - name: inspec-generated
+
+run:
+    path: magic-modules-branched/.ci/magic-modules/generate-inspec.sh

.ci/magic-modules/point-to-submodules.sh

@@ -59,6 +59,14 @@ if [ "$ANSIBLE_ENABLED" = "true" ]; then
   git add build/ansible
 fi
 
+if [ "$INSPEC_ENABLED" = "true" ]; then
+  git config -f .gitmodules submodule.build/inspec.branch "$BRANCH"
+  git config -f .gitmodules submodule.build/inspec.url "[email protected]:$GH_USERNAME/inspec-gcp.git"
+  git submodule sync build/inspec
+  ssh-agent bash -c "ssh-add ~/github_private_key; git submodule update --remote --init build/inspec"
+  git add build/inspec
+fi
+
 # Commit those changes so that they can be tested in the next phase.
 git add .gitmodules
 git config --global user.email "[email protected]"

.ci/magic-modules/point-to-submodules.yml

@@ -24,6 +24,7 @@ params:
   CREDS: ""
   TERRAFORM_ENABLED: false
   ANSIBLE_ENABLED: false
+  INSPEC_ENABLED: false
   PUPPET_MODULES: ""
   CHEF_MODULES: ""
 

.ci/unit-tests/inspec.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+echo 'TODO slevenick write tests'

.ci/unit-tests/inspec.yml

@@ -0,0 +1,14 @@
+platform: linux
+inputs:
+  - name: magic-modules
+image_resource:
+  type: docker-image
+  source:
+    repository: ruby
+    tag: '2.5'
+run:
+  path: magic-modules/.ci/unit-tests/inspec.sh
+params:
+  PRODUCT: ""
+  PROVIDER: inspec
+  EXCLUDE_PATTERN: ""

.ci/vars.tmpl

@@ -10,7 +10,7 @@ build/{{repo}}/{{name}}
 {% set chef_submodules = names_as_list('chef', chef_modules).split() %}
 {%
   set all_submodules = puppet_submodules + chef_submodules +
-     (['build/terraform'] + ['build/ansible'])
+     (['build/terraform'] + ['build/ansible'] + ['build/inspec'])
 %}
 {% set all_submodules_yaml_format = '[' + ','.join(all_submodules) + ']' %}
 {% set chef_test_excludes = {

.github/PULL_REQUEST_TEMPLATE.md

@@ -40,3 +40,4 @@ this PR's changes from the commit messages for downstream commits.
 ### [chef-sql]
 ### [chef-storage]
 ## [ansible]
+## [inspec]

.gitmodules

@@ -81,3 +81,6 @@
 [submodule "build/chef/iam"]
 	path = build/chef/iam
 	url = [email protected]:GoogleCloudPlatform/chef-google-iam.git
+[submodule "build/inspec"]
+	path = build/inspec
+	url = [email protected]:modular-magician/inspec-gcp.git

Rakefile

@@ -19,7 +19,8 @@ PROVIDER_FOLDERS = {
   ansible: 'build/ansible',
   puppet: 'build/puppet/%<mod>s',
   chef: 'build/chef/%<mod>s',
-  terraform: 'build/terraform'
+  terraform: 'build/terraform',
+  inspec: 'build/inspec'
 }.freeze
 
 # Requires

compiler.rb

@@ -31,6 +31,7 @@
 require 'provider/chef'
 require 'provider/chef/bundle'
 require 'provider/example'
+require 'provider/inspec'
 require 'provider/puppet'
 require 'provider/puppet/bundle'
 require 'provider/terraform'

products/compute/inspec.yaml

@@ -0,0 +1,29 @@
+# Copyright 2017 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--- !ruby/object:Provider::Inspec::Config
+manifest: !ruby/object:Provider::Inspec::Manifest
+  version: '0.1.0'
+  source: 'FIXME'
+  issues: 'FIXME'
+  summary: 'InSpec resources for verifying GCP infrastructure'
+  description: |
+    InSpec resources for verifying GCP infrastructure
+files: !ruby/object:Provider::Config::Files
+style:
+functions:
+changelog:
+  - !ruby/object:Provider::Config::Changelog
+    version: '0.1.0'
+    date: 2017-10-04T10:00:00-0700
+    general: 'Initial release'

provider/inspec.rb

@@ -0,0 +1,71 @@
+# Copyright 2017 Google Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'provider/config'
+require 'provider/core'
+require 'provider/inspec/manifest'
+require 'provider/inspec/resource_override'
+require 'provider/inspec/property_override'
+
+module Provider
+  # Code generator for Example Cookbooks that manage Google Cloud Platform
+  # resources.
+  class Inspec < Provider::Core
+    # Settings for the provider
+    class Config < Provider::Config
+      attr_reader :manifest
+      def provider
+        Provider::Inspec
+      end
+
+      def resource_override
+        Provider::Inspec::ResourceOverride
+      end
+
+      def property_override
+        Provider::Inspec::PropertyOverride
+      end
+    end
+
+    # This function uses the resource templates to create singular and plural
+    # resources that can be used by InSpec
+    def generate_resource(data)
+      target_folder = File.join(data[:output_folder], 'inspec')
+      FileUtils.mkpath target_folder
+      name = data[:object].name.underscore
+      generate_resource_file data.clone.merge(
+        default_template: 'templates/inspec/singular_resource.erb',
+        out_file: File.join(target_folder, "google_#{data[:product_name]}_#{name}.rb")
+      )
+      generate_resource_file data.clone.merge(
+        default_template: 'templates/inspec/plural_resource.erb',
+        out_file: File.join(target_folder, "google_#{data[:product_name]}_#{name}s.rb")
+      )
+    end
+
+    # TODO?
+    def generate_resource_tests(data) end
+
+    def generate_base_property(data) end
+
+    def generate_simple_property(type, data) end
+
+    def generate_typed_array(data, prop) end
+
+    def emit_resourceref_object(data) end
+
+    def emit_nested_object(data) end
+
+    def generate_network_datas(data, object) end
+  end
+end