Initial functional downstream diff generator. (#2848)

Merged PR #2848.
GoogleCloudPlatform · Dec 17, 2019 · f3b37e5 · f3b37e5
1 parent dd85d49
commit f3b37e5
Show file tree

Hide file tree

Showing 5 changed files with 353 additions and 0 deletions.
diff --git a/.ci/containers/downstream-builder/Dockerfile b/.ci/containers/downstream-builder/Dockerfile
@@ -0,0 +1,68 @@
+from golang:1.13-stretch as resource
+
+SHELL ["/bin/bash", "-c"]
+
+RUN go get golang.org/x/tools/cmd/goimports
+
+# Set up Github SSH cloning.
+RUN ssh-keyscan github.com >> /known_hosts
+RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
+
+ENV GOFLAGS "-mod=vendor"
+ENV GO111MODULE "off"
+
+# Install Ruby from source.
+RUN apt-get update
+RUN apt-get install -y bzip2 libssl-dev libreadline-dev zlib1g-dev
+RUN git clone https://github.com/rbenv/rbenv.git /rbenv
+ENV PATH /rbenv/bin:/root/.rbenv/shims:$PATH
+
+ENV RUBY_VERSION 2.6.0
+ENV RUBYGEMS_VERSION 3.0.2
+ENV BUNDLER_VERSION 1.17.0
+
+RUN /rbenv/bin/rbenv init || true
+RUN eval "$(rbenv init -)"
+RUN mkdir -p "$(rbenv root)"/plugins
+RUN git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
+
+RUN rbenv install $RUBY_VERSION
+RUN rbenv global $RUBY_VERSION
+RUN rbenv rehash
+
+RUN gem update --system "$RUBYGEMS_VERSION"
+RUN gem install bundler --version "$BUNDLER_VERSION" --force
+
+RUN apt-get update
+RUN apt-get install -y git build-essential libbz2-dev libssl-dev libreadline-dev \
+                        libffi-dev libsqlite3-dev tk-dev libpng-dev libfreetype6-dev \
+                        make build-essential libssl-dev zlib1g-dev libbz2-dev \
+                        libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev \
+                        xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/magic-modules/master/Gemfile
+RUN wget https://raw.githubusercontent.com/GoogleCloudPlatform/magic-modules/master/Gemfile.lock
+RUN bundle install
+RUN rbenv rehash
+RUN rm Gemfile Gemfile.lock
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/root/.pyenv/bin:${PATH}"
+RUN eval "$(pyenv init -)"
+RUN eval "$(pyenv virtualenv-init -)"
+RUN pyenv install 3.6.8
+RUN pyenv install 2.7.13
+RUN pyenv rehash
+ENV PATH="/root/.pyenv/shims:${PATH}"
+RUN pyenv global 2.7.13 3.6.8
+RUN pip install beautifulsoup4 mistune
+RUN pip3 install black
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN git config --global user.name "Modular Magician"
+RUN git config --global user.email "[email protected]"
+
+ADD generate_downstream.sh /generate_downstream.sh
+
+ENTRYPOINT ["/generate_downstream.sh"]
diff --git a/.ci/containers/downstream-builder/generate_downstream.sh b/.ci/containers/downstream-builder/generate_downstream.sh
@@ -0,0 +1,88 @@
+#! /bin/bash
+
+set -e
+
+function clone_repo() {
+    if [ "$REPO" == "terraform" ]; then
+        if [ "$VERSION" == "ga" ]; then
+            GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/terraform-providers/terraform-provider-google
+            SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-provider-google
+            LOCAL_PATH=$GOPATH/src/github.com/terraform-providers/terraform-provider-google
+        elif [ "$VERSION" == "beta" ]; then
+            GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/terraform-providers/terraform-provider-google-beta
+            SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-provider-google-beta
+            LOCAL_PATH=$GOPATH/src/github.com/terraform-providers/terraform-provider-google-beta
+        else
+            echo "Unrecognized version $VERSION"
+            exit 1
+        fi
+    elif [ "$REPO" == "tf-conversion" ]; then
+        GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/GoogleCloudPlatform/terraform-google-conversion
+        SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-google-conversion
+        LOCAL_PATH=$GOPATH/src/github.com/GoogleCloudPlatform/terraform-google-conversion
+    elif [ "$REPO" == "ansible" ]; then
+        GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/ansible-collections/ansible_collections_google
+        SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/ansible_collections_google
+        LOCAL_PATH=$PWD/../ansible
+    elif [ "$REPO" == "inspec" ]; then
+        GITHUB_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/inspec-gcp
+        SCRATCH_PATH=$GITHUB_PATH
+        LOCAL_PATH=$PWD/../inspec
+    else
+        echo "Unrecognized repo $REPO"
+        exit 1
+    fi
+    mkdir -p "$(dirname $LOCAL_PATH)"
+    git clone $GITHUB_PATH $LOCAL_PATH
+}
+
+if [ $# -lt 4 ]; then
+    echo "Usage: $0 (build|diff) (terraform|tf-conversion|ansible|inspec) (ga|beta) (pr number)"
+    exit 1
+fi
+if [ -z "$GITHUB_TOKEN" ]; then
+    echo "Did not provide GITHUB_TOKEN environment variable."
+    exit 1
+fi
+
+COMMAND=$1
+REPO=$2
+VERSION=$3
+PR_NUMBER=$4
+
+clone_repo
+
+git config --global user.name "Modular Magician"
+git config --global user.email "[email protected]"
+
+if [ "$COMMAND" == "head" ]; then
+    BRANCH=auto-pr-$PR_NUMBER
+elif [ "$COMMAND" == "base" ]; then
+    git checkout HEAD~
+    BRANCH=auto-pr-$PR_NUMBER-old
+fi
+
+if [ "$REPO" == "terraform" ]; then
+    pushd $LOCAL_PATH
+    go get -v
+    find . -type f -not -wholename "./.git*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name ".go-version" -not -name ".hashibot.hcl" -not -name "tools.go"  -exec git rm {} \;
+    popd
+elif [ "$REPO" == "tf-conversion" ]; then
+    pushd $LOCAL_PATH
+    go get -v ./google
+    popd
+fi
+
+if [ "$REPO" == "tf-conversion" ]; then
+    bundle exec compiler -a -e terraform -f validator -o $LOCAL_PATH -v $VERSION
+else
+    bundle exec compiler -a -e $REPO -o $LOCAL_PATH -v $VERSION
+fi
+
+
+pushd $LOCAL_PATH
+git add .
+git checkout -b $BRANCH
+git commit -m "New generated code for PR $PR_NUMBER." || true
+git push $SCRATCH_PATH $BRANCH -f
+popd
diff --git a/.ci/containers/github-differ/Dockerfile b/.ci/containers/github-differ/Dockerfile
@@ -0,0 +1,6 @@
+FROM alpine/git
+
+RUN apk add --no-cache curl
+RUN apk add --no-cache bash
+ADD generate_comment.sh /generate_comment.sh
+ENTRYPOINT ["/generate_comment.sh"]
diff --git a/.ci/containers/github-differ/generate_comment.sh b/.ci/containers/github-differ/generate_comment.sh
@@ -0,0 +1,90 @@
+#! /bin/bash
+
+set -e
+
+
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 pr-number"
+    exit 1
+fi
+if [ -z "$GITHUB_TOKEN" ]; then
+    echo "Did not provide GITHUB_TOKEN environment variable."
+    exit 1
+fi
+
+PR_NUMBER=$1
+NEW_BRANCH=auto-pr-$PR_NUMBER
+OLD_BRANCH=auto-pr-$PR_NUMBER-old
+TPG_SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-provider-google
+TPG_LOCAL_PATH=$PWD/../tpg
+TPGB_SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-provider-google-beta
+TPGB_LOCAL_PATH=$PWD/../tpgb
+TFC_SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/terraform-google-conversion
+TFC_LOCAL_PATH=$PWD/../tfc
+ANSIBLE_SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/ansible_collections_google
+ANSIBLE_LOCAL_PATH=$PWD/../ansible
+INSPEC_SCRATCH_PATH=https://modular-magician:$GITHUB_TOKEN@github.com/modular-magician/inspec-gcp
+INSPEC_LOCAL_PATH=$PWD/../inspec
+
+DIFFS=""
+NEWLINE=$'\n'
+
+# TPG
+mkdir -p $TPG_LOCAL_PATH
+git clone -b $NEW_BRANCH $TPG_SCRATCH_PATH $TPG_LOCAL_PATH
+pushd $TPG_LOCAL_PATH
+git fetch origin $OLD_BRANCH
+if ! git diff --exit-code origin/$NEW_BRANCH origin/$OLD_BRANCH; then
+    DIFFS="${DIFFS}${NEWLINE}Terraform GA: [Diff](https://github.com/modular-magician/terraform-provider-google/compare/$OLD_BRANCH..$NEW_BRANCH)"
+fi
+popd
+
+# TPGB
+mkdir -p $TPGB_LOCAL_PATH
+git clone -b $NEW_BRANCH $TPGB_SCRATCH_PATH $TPGB_LOCAL_PATH
+pushd $TPGB_LOCAL_PATH
+git fetch origin $OLD_BRANCH
+if ! git diff --exit-code origin/$NEW_BRANCH origin/$OLD_BRANCH; then
+    DIFFS="${DIFFS}${NEWLINE}Terraform Beta: [Diff](https://github.com/modular-magician/terraform-provider-google-beta/compare/$OLD_BRANCH..$NEW_BRANCH)"
+fi
+popd
+
+# Ansible
+mkdir -p $ANSIBLE_LOCAL_PATH
+git clone -b $NEW_BRANCH $ANSIBLE_SCRATCH_PATH $ANSIBLE_LOCAL_PATH
+pushd $ANSIBLE_LOCAL_PATH
+git fetch origin $OLD_BRANCH
+if ! git diff --exit-code origin/$NEW_BRANCH origin/$OLD_BRANCH; then
+    DIFFS="${DIFFS}${NEWLINE}Ansible: [Diff](https://github.com/modular-magician/ansible_collections_google/compare/$OLD_BRANCH..$NEW_BRANCH)"
+fi
+popd
+
+# TF Conversion
+mkdir -p $TFC_LOCAL_PATH
+git clone -b $NEW_BRANCH $TFC_SCRATCH_PATH $TFC_LOCAL_PATH
+pushd $TFC_LOCAL_PATH
+git fetch origin $OLD_BRANCH
+if ! git diff --exit-code origin/$NEW_BRANCH origin/$OLD_BRANCH; then
+    DIFFS="${DIFFS}${NEWLINE}TF Conversion: [Diff](https://github.com/modular-magician/terraform-google-conversion/compare/$OLD_BRANCH..$NEW_BRANCH)"
+fi
+popd
+
+# Inspec
+mkdir -p $INSPEC_LOCAL_PATH
+git clone -b $NEW_BRANCH $INSPEC_SCRATCH_PATH $INSPEC_LOCAL_PATH
+pushd $INSPEC_LOCAL_PATH
+git fetch origin $OLD_BRANCH
+if ! git diff --exit-code origin/$NEW_BRANCH origin/$OLD_BRANCH; then
+    DIFFS="${DIFFS}${NEWLINE}Inspec: [Diff](https://github.com/modular-magician/inspec-gcp/compare/$OLD_BRANCH..$NEW_BRANCH)"
+fi
+popd
+
+if [ -z "$DIFFS" ]; then
+    DIFFS="Hi!  I'm the modular magician.  Your PR hasn't generated any diffs, but I'll let you know if a future commit does."
+else
+    DIFFS="Hi!  I'm the modular magician.  Your PR generated some diffs in downstreams - here they are.$NEWLINE# Diff report:$NEWLINE$NEWLINE$DIFFS"
+fi
+
+curl -s -H "Authorization: token ${GITHUB_TOKEN}" \
+     -X POST -d "{\"body\": \"$DIFFS\"}" \
+      "https://api.github.com/repos/GoogleCloudPlatform/magic-modules/issues/${PR_NUMBER}/comments"
diff --git a/.ci/gcb-generate-diffs.yml b/.ci/gcb-generate-diffs.yml
@@ -0,0 +1,101 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'head'
+          - 'terraform'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'base'
+          - 'terraform'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'head'
+          - 'terraform'
+          - 'beta'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'base'
+          - 'terraform'
+          - 'beta'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'head'
+          - 'ansible'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'base'
+          - 'ansible'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'head'
+          - 'inspec'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'base'
+          - 'inspec'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'head'
+          - 'tf-conversion'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/downstream-builder'
+      secretEnv: ["GITHUB_TOKEN"]
+      waitFor: ["-"]
+      args:
+          - 'base'
+          - 'tf-conversion'
+          - 'ga'
+          - $_PR_NUMBER
+
+    - name: 'gcr.io/graphite-docker-images/github-differ'
+      secretEnv: ["GITHUB_TOKEN"]
+      args:
+          - $_PR_NUMBER
+
+secrets:
+    - kmsKeyName: projects/graphite-docker-images/locations/global/keyRings/token-keyring/cryptoKeys/github-token
+      secretEnv:
+          GITHUB_TOKEN: CiQADkR4NnCVXo1OLSWFuPX7eSiifaOfQVzSYmKi2jZdVbKlfYMSUQBfF82vNAgpvSVyhzM8JsQaP6Oky0SAdoR5fPED5cU3qxsCB9wArmdGcgQoRzP7S6jEWHRcvxv/xauznjkJQMWCORzcbUbk6T7k80bdo2mpqw==