Private Cluster Configuration

.gitignore

@@ -33,7 +33,6 @@ crash.log
 
 # Kitchen files
 **/inspec.lock
-**.gem
 **/.kitchen
 **/.kitchen.local.yml
 **/Gemfile.lock
@@ -47,3 +46,5 @@ crash.log
 
 test/integration/gcloud/config.sh
 test/integration/tmp
+
+credentials.json

.kitchen.yml

@@ -0,0 +1,124 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+driver:
+  name: "terraform"
+  command_timeout: 1800
+
+provisioner:
+  name: "terraform"
+
+platforms:
+  - name: local
+
+suites:
+  - name: "deploy_service"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/deploy_service
+      variable_files:
+        - test/fixtures/deploy_service/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: deploy_service
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "node_pool"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/node_pool
+      variable_files:
+        - test/fixtures/node_pool/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: node_pool
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_regional"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/simple_regional
+      variable_files:
+        - test/fixtures/simple_regional/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_regional
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_zonal"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/simple_zonal
+      variable_files:
+        - test/fixtures/simple_zonal/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_zonal
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_regional_private"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/simple_regional_private
+      variable_files:
+        - test/fixtures/simple_regional_private/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_regional_private
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "simple_zonal_private"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/simple_zonal_private
+      variable_files:
+        - test/fixtures/simple_zonal_private/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: simple_zonal_private
+          backend: local
+    provisioner:
+      name: terraform
+  - name: "stub_domains"
+    driver:
+      name: "terraform"
+      command_timeout: 1800
+      root_module_directory: examples/stub_domains
+      variable_files:
+        - test/fixtures/stub_domains/terraform.tfvars
+    verifier:
+      name: terraform
+      systems:
+        - name: stub_domains
+          backend: local
+    provisioner:
+      name: terraform

.ruby-version

@@ -0,0 +1 @@
+2.4.2

CHANGELOG.md

@@ -8,6 +8,9 @@ project adheres to [Semantic Versioning](http://semver.org/).
 ## [Unreleased]
 ### Changed
 
+* Added suport for private clusters. #21
+* Migrated to [google-beta provider](https://github.com/terraform-providers/terraform-provider-google-beta) to support private clusters. #21
+
 ## [v0.3.0] - 2018-10-10
 ### Changed
 * Updated network/subnetwork lookup to use data source. #16

test/integration/gcloud/Gemfile → Gemfile

@@ -15,8 +15,7 @@
 ruby '2.4.2'
 
 source 'https://rubygems.org/' do
-  gem 'googleauth'
-  gem 'google-api-client'
-  gem 'kitchen-terraform', '~> 3.3'
-  gem 'kitchen-inspec', :git => 'https://github.com/inspec/kitchen-inspec.git', :ref => '0590f1b'
+  gem 'kitchen-terraform', '~> 4.0.3'
+  gem 'kubeclient'
+  gem 'rest-client'
 end

Makefile

@@ -15,6 +15,18 @@
 # Make will use bash instead of sh
 SHELL := /usr/bin/env bash
 
+# Docker build config variables
+BUILD_TERRAFORM_VERSION ?= 0.11.8
+BUILD_CLOUD_SDK_VERSION ?= 216.0.0
+BUILD_PROVIDER_GOOGLE_VERSION ?= 1.17.1
+BUILD_PROVIDER_GSUITE_VERSION ?= 0.1.8
+DOCKER_IMAGE_TERRAFORM := cftk/terraform
+DOCKER_TAG_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
+BUILD_RUBY_VERSION := 2.4.2
+DOCKER_IMAGE_KITCHEN_TERRAFORM := cftk/kitchen_terraform
+DOCKER_TAG_KITCHEN_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION}
+TEST_CONFIG_FILE_LOCATION := "./test/fixtures/config.sh"
+
 # All is the first target in the file so it will get picked up when you just run 'make' on its own
 all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs
 
@@ -63,20 +75,84 @@ check_headers:
 	@echo "Checking file headers"
 	@python test/verify_boilerplate.py
 
+# Integration tests
+.PHONY: test_integration
+test_integration:
+	source ${TEST_CONFIG_FILE_LOCATION}
+	bundle install
+	bundle exec kitchen create
+	bundle exec kitchen converge
+	bundle exec kitchen converge
+	@echo "Waiting ${GCE_INSTANCE_INIT_WAIT_TIME} seconds for load balancer to come online..."
+	bundle exec kitchen verify
+	bundle exec kitchen destroy
+
 .PHONY: generate_docs
 generate_docs:
 	@source test/make.sh && generate_docs
 
-# Integration tests
-
-.PHONY: regional_test_integration
-regional_test_integration:
-	./test/integration/gcloud/run.sh regional
-
-.PHONY: zonal_test_integration
-zonal_test_integration:
-	./test/integration/gcloud/run.sh zonal
-
-.PHONY: test_integration
-test_integration: regional_test_integration zonal_test_integration
-	@echo "Running tests for regional and zonal clusters"
+# Versioning
+.PHONY: version
+version:
+	@source helpers/version-repo.sh
+
+# Build Docker
+.PHONY: docker_build_terraform
+docker_build_terraform:
+	docker build -f build/docker/terraform/Dockerfile \
+		--build-arg BUILD_TERRAFORM_VERSION=${BUILD_TERRAFORM_VERSION} \
+		--build-arg BUILD_CLOUD_SDK_VERSION=${BUILD_CLOUD_SDK_VERSION} \
+		--build-arg BUILD_PROVIDER_GOOGLE_VERSION=${BUILD_PROVIDER_GOOGLE_VERSION} \
+		--build-arg BUILD_PROVIDER_GSUITE_VERSION=${BUILD_PROVIDER_GSUITE_VERSION} \
+		-t ${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM} .
+
+.PHONY: docker_build_kitchen_terraform
+docker_build_kitchen_terraform:
+	docker build -f build/docker/kitchen_terraform/Dockerfile \
+		--build-arg BUILD_TERRAFORM_IMAGE="${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM}" \
+		--build-arg BUILD_RUBY_VERSION="${BUILD_RUBY_VERSION}" \
+		-t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} .
+
+# Run docker
+.PHONY: docker_run
+docker_run:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash
+
+.PHONY: docker_create
+docker_create: docker_build_terraform docker_build_kitchen_terraform
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen create"
+
+.PHONY: docker_converge
+docker_converge:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen converge && kitchen converge"
+
+.PHONY: docker_verify
+docker_verify:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen verify"
+
+.PHONY: docker_destroy
+docker_destroy:
+	docker run --rm -it \
+		-v $(CURDIR):/cftk/workdir \
+		${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \
+		/bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen destroy"
+
+.PHONY: test_integration_docker
+test_integration_docker: docker_create docker_converge docker_verify docker_destroy
+	@echo "Running test-kitchen tests in docker"
+
+.PHONY: prepare_test_variables
+prepare_test_variables:
+	@source test/make.sh && prepare_test_variables

README.md

@@ -109,6 +109,10 @@ Then perform the following commands on the root folder:
 | node_pools_taints | Map of lists containing node taints by node-pool name | map | `<map>` | no |
 | node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no |
 | non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `<list>` | no |
+| private | (Beta) Provision as a private cluster | string | `false` | no |
+| private_cluster_config_enable_private_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `false` | no |
+| private_cluster_config_enable_private_nodes | (Beta) Whether nodes have internal IP addresses only | string | `false` | no |
+| private_cluster_config_master_ipv4_cidr_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `10.0.0.0/28` | no |
 | project_id | The project ID to host the cluster in (required) | string | - | yes |
 | region | The region to host the cluster in (required) | string | - | yes |
 | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no |
@@ -185,7 +189,6 @@ The project has the following folders and files:
 ### Requirements
 - [bundler](https://github.com/bundler/bundler)
 - [gcloud](https://cloud.google.com/sdk/install)
-- [jq](https://stedolan.github.io/jq/) 1.5
 - [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0
 
 ### Autogeneration of documentation from .tf files
@@ -208,12 +211,13 @@ The tests will do the following:
 - Perform `kitchen validate` command
   - Performs inspec tests.
     - Shell out to `gcloud` to validate expected resources in GCP.
-    - Shell out to `kubectl` to validate expected resource in Kubernetes.
-    - Shell out to `terraform` to validate outputs.
-- Permos `kitchen destroy` command
+    - Interrogate the cluster to validate expected resource in Kubernetes.
+- Perform `kitchen destroy` command
   - Performs a `terraform destroy -force`
 
-You can use the following command to run the integration test in the root folder
+To configure the integration tests, run `make prepare_test_variables` and edit each of the files it outputs to reflect your existing GCP setup.
+
+You can then use the following command to run the integration test in the root folder
 
   `make test_integration`
 

auth.tf

@@ -17,7 +17,9 @@
 /******************************************
   Retrieve authentication token
  *****************************************/
-data "google_client_config" "default" {}
+data "google_client_config" "default" {
+  provider = "google-beta"
+}
 
 /******************************************
   Configure provider

build/docker/kitchen_terraform/Dockerfile

@@ -0,0 +1,63 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BUILD_TERRAFORM_IMAGE
+ARG BUILD_RUBY_VERSION
+# hadolint ignore=DL3006
+FROM $BUILD_TERRAFORM_IMAGE as cfkt_terraform
+
+
+
+FROM ruby:$BUILD_RUBY_VERSION-alpine
+
+RUN apk add --no-cache \
+        bash=4.3.42-r5 \
+        curl=7.60.0-r1 \
+        git=2.8.6-r0 \
+        g++=5.3.0-r0 \
+        jq=1.5-r2 \
+        make=4.1-r1 \
+        musl-dev=1.1.14-r16 \
+        python=2.7.14-r0 \
+        python-dev=2.7.14-r0 \
+        py-pip=8.1.2-r0
+
+SHELL ["/bin/bash", "-c"]
+
+ENV APP_BASE_DIR="/cftk"
+
+COPY --from=cfkt_terraform $APP_BASE_DIR $APP_BASE_DIR
+
+ENV HOME="$APP_BASE_DIR/home"
+ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH
+ENV GOOGLE_APPLICATION_CREDENTIALS="$CREDENTIALS_PATH" \
+    CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$CREDENTIALS_PATH"
+
+# Fix base64 inconsistency
+SHELL ["/bin/bash", "-c"]
+RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc
+
+RUN terraform --version && \
+    gcloud --version && \
+    ruby --version && \
+    bundle --version
+
+COPY ./Gemfile /opt/kitchen/
+
+WORKDIR /opt/kitchen
+RUN bundle install
+
+RUN gcloud components install beta --quiet
+
+WORKDIR $APP_BASE_DIR/workdir