From d71c70bc61cad51ab51eb5109bbf2020e30739c3 Mon Sep 17 00:00:00 2001 From: Jason Berlinsky Date: Tue, 23 Oct 2018 11:25:22 -0400 Subject: [PATCH 1/5] Restructure tests --- .gitignore | 3 +- .kitchen.yml | 96 +++++ .ruby-version | 1 + test/integration/gcloud/Gemfile => Gemfile | 7 +- Makefile | 102 ++++- README.md | 10 +- build/docker/kitchen_terraform/Dockerfile | 63 ++++ build/docker/terraform/Dockerfile | 100 +++++ examples/deploy_service/README.md | 43 ++- examples/deploy_service/main.tf | 7 +- examples/deploy_service/outputs.tf | 60 ++- examples/deploy_service/variables.tf | 4 + examples/node_pool/README.md | 44 ++- examples/node_pool/main.tf | 13 +- examples/node_pool/outputs.tf | 55 ++- examples/node_pool/variables.tf | 6 +- examples/shared_vpc/README.md | 18 - examples/shared_vpc/main.tf | 35 -- examples/shared_vpc/outputs.tf | 36 -- examples/shared_vpc/variables.tf | 43 --- examples/simple_regional/README.md | 41 +- examples/simple_regional/main.tf | 9 +- examples/simple_regional/outputs.tf | 44 ++- examples/simple_regional/variables.tf | 4 + examples/simple_zonal/README.md | 42 ++- examples/simple_zonal/main.tf | 8 +- examples/simple_zonal/outputs.tf | 45 ++- examples/simple_zonal/variables.tf | 4 + examples/stub_domains/README.md | 43 ++- examples/stub_domains/main.tf | 10 +- examples/stub_domains/outputs.tf | 55 ++- examples/stub_domains/variables.tf | 9 + helpers/combine_docfiles.py | 16 +- main.tf | 2 +- networks.tf | 15 + .../.kitchen.yml => fixtures/config.sh} | 23 +- .../deploy_service/terraform.tfvars.sample | 8 + .../node_pool/terraform.tfvars.sample | 8 + .../simple_regional/terraform.tfvars.sample | 7 + .../simple_zonal/terraform.tfvars.sample | 8 + .../stub_domains/terraform.tfvars.sample | 9 + .../deploy_service/controls/gcloud.rb | 64 ++++ .../deploy_service/controls/kubectl.rb | 67 ++++ test/integration/deploy_service/inspec.yml | 40 ++ test/integration/gcloud/.gitignore | 1 - test/integration/gcloud/run.sh | 352 ------------------ test/integration/gcloud/sample.sh | 44 --- .../test/integration/default/inspec/gcloud.rb | 127 ------- .../integration/default/inspec/kubectl.rb | 38 -- .../integration/default/inspec/terraform.rb | 102 ----- .../gcloud/test/support/google_cloud.rb | 22 -- test/integration/gcloud/zonal_config.sh | 25 -- test/integration/node_pool/controls/gcloud.rb | 165 ++++++++ .../integration/node_pool/controls/kubectl.rb | 87 +++++ test/integration/node_pool/inspec.yml | 40 ++ .../simple_regional/controls/gcloud.rb | 146 ++++++++ test/integration/simple_regional/inspec.yml | 37 ++ .../simple_zonal/controls/gcloud.rb | 147 ++++++++ test/integration/simple_zonal/inspec.yml | 37 ++ .../stub_domains/controls/gcloud.rb | 72 ++++ .../stub_domains/controls/kubectl.rb | 88 +++++ test/integration/stub_domains/inspec.yml | 40 ++ test/make.sh | 15 +- 63 files changed, 1906 insertions(+), 1006 deletions(-) create mode 100644 .kitchen.yml create mode 100644 .ruby-version rename test/integration/gcloud/Gemfile => Gemfile (77%) create mode 100644 build/docker/kitchen_terraform/Dockerfile create mode 100644 build/docker/terraform/Dockerfile delete mode 100644 examples/shared_vpc/README.md delete mode 100644 examples/shared_vpc/main.tf delete mode 100644 examples/shared_vpc/outputs.tf delete mode 100644 examples/shared_vpc/variables.tf rename test/{integration/gcloud/.kitchen.yml => fixtures/config.sh} (74%) mode change 100644 => 100755 create mode 100644 test/fixtures/deploy_service/terraform.tfvars.sample create mode 100644 test/fixtures/node_pool/terraform.tfvars.sample create mode 100644 test/fixtures/simple_regional/terraform.tfvars.sample create mode 100644 test/fixtures/simple_zonal/terraform.tfvars.sample create mode 100644 test/fixtures/stub_domains/terraform.tfvars.sample create mode 100644 test/integration/deploy_service/controls/gcloud.rb create mode 100644 test/integration/deploy_service/controls/kubectl.rb create mode 100644 test/integration/deploy_service/inspec.yml delete mode 100644 test/integration/gcloud/.gitignore delete mode 100755 test/integration/gcloud/run.sh delete mode 100644 test/integration/gcloud/sample.sh delete mode 100644 test/integration/gcloud/test/integration/default/inspec/gcloud.rb delete mode 100644 test/integration/gcloud/test/integration/default/inspec/kubectl.rb delete mode 100644 test/integration/gcloud/test/integration/default/inspec/terraform.rb delete mode 100644 test/integration/gcloud/test/support/google_cloud.rb delete mode 100644 test/integration/gcloud/zonal_config.sh create mode 100644 test/integration/node_pool/controls/gcloud.rb create mode 100644 test/integration/node_pool/controls/kubectl.rb create mode 100644 test/integration/node_pool/inspec.yml create mode 100644 test/integration/simple_regional/controls/gcloud.rb create mode 100644 test/integration/simple_regional/inspec.yml create mode 100644 test/integration/simple_zonal/controls/gcloud.rb create mode 100644 test/integration/simple_zonal/inspec.yml create mode 100644 test/integration/stub_domains/controls/gcloud.rb create mode 100644 test/integration/stub_domains/controls/kubectl.rb create mode 100644 test/integration/stub_domains/inspec.yml diff --git a/.gitignore b/.gitignore index 2ad2fdfd88..11b400bcb4 100644 --- a/.gitignore +++ b/.gitignore @@ -33,7 +33,6 @@ crash.log # Kitchen files **/inspec.lock -**.gem **/.kitchen **/.kitchen.local.yml **/Gemfile.lock @@ -47,3 +46,5 @@ crash.log test/integration/gcloud/config.sh test/integration/tmp + +credentials.json diff --git a/.kitchen.yml b/.kitchen.yml new file mode 100644 index 0000000000..4dbcbf0265 --- /dev/null +++ b/.kitchen.yml @@ -0,0 +1,96 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +driver: + name: "terraform" + command_timeout: 1800 + +provisioner: + name: "terraform" + +platforms: + - name: local + +suites: + - name: "deploy_service" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/deploy_service + variable_files: + - test/fixtures/deploy_service/terraform.tfvars + verifier: + name: terraform + systems: + - name: deploy_service + backend: local + provisioner: + name: terraform + - name: "node_pool" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/node_pool + variable_files: + - test/fixtures/node_pool/terraform.tfvars + verifier: + name: terraform + systems: + - name: node_pool + backend: local + provisioner: + name: terraform + - name: "simple_regional" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/simple_regional + variable_files: + - test/fixtures/simple_regional/terraform.tfvars + verifier: + name: terraform + systems: + - name: simple_regional + backend: local + provisioner: + name: terraform + - name: "simple_zonal" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/simple_zonal + variable_files: + - test/fixtures/simple_zonal/terraform.tfvars + verifier: + name: terraform + systems: + - name: simple_zonal + backend: local + provisioner: + name: terraform + - name: "stub_domains" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/stub_domains + variable_files: + - test/fixtures/stub_domains/terraform.tfvars + verifier: + name: terraform + systems: + - name: stub_domains + backend: local + provisioner: + name: terraform diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000000..8e8299dcc0 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +2.4.2 diff --git a/test/integration/gcloud/Gemfile b/Gemfile similarity index 77% rename from test/integration/gcloud/Gemfile rename to Gemfile index cc5ca265d9..35e4ef7d1f 100644 --- a/test/integration/gcloud/Gemfile +++ b/Gemfile @@ -15,8 +15,7 @@ ruby '2.4.2' source 'https://rubygems.org/' do - gem 'googleauth' - gem 'google-api-client' - gem 'kitchen-terraform', '~> 3.3' - gem 'kitchen-inspec', :git => 'https://github.com/inspec/kitchen-inspec.git', :ref => '0590f1b' + gem 'kitchen-terraform', '~> 4.0.3' + gem 'kubeclient' + gem 'rest-client' end diff --git a/Makefile b/Makefile index de5a344569..6e169190fc 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,18 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash +# Docker build config variables +BUILD_TERRAFORM_VERSION ?= 0.11.8 +BUILD_CLOUD_SDK_VERSION ?= 216.0.0 +BUILD_PROVIDER_GOOGLE_VERSION ?= 1.17.1 +BUILD_PROVIDER_GSUITE_VERSION ?= 0.1.8 +DOCKER_IMAGE_TERRAFORM := cftk/terraform +DOCKER_TAG_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION} +BUILD_RUBY_VERSION := 2.4.2 +DOCKER_IMAGE_KITCHEN_TERRAFORM := cftk/kitchen_terraform +DOCKER_TAG_KITCHEN_TERRAFORM ?= ${BUILD_TERRAFORM_VERSION}_${BUILD_CLOUD_SDK_VERSION}_${BUILD_PROVIDER_GOOGLE_VERSION}_${BUILD_PROVIDER_GSUITE_VERSION} +TEST_CONFIG_FILE_LOCATION := "./test/fixtures/config.sh" + # All is the first target in the file so it will get picked up when you just run 'make' on its own all: check_shell check_python check_golang check_terraform check_docker check_base_files test_check_headers check_headers check_trailing_whitespace generate_docs @@ -63,20 +75,84 @@ check_headers: @echo "Checking file headers" @python test/verify_boilerplate.py +# Integration tests +.PHONY: test_integration +test_integration: + source ${TEST_CONFIG_FILE_LOCATION} + bundle install + bundle exec kitchen create + bundle exec kitchen converge + bundle exec kitchen converge + @echo "Waiting ${GCE_INSTANCE_INIT_WAIT_TIME} seconds for load balancer to come online..." + bundle exec kitchen verify + bundle exec kitchen destroy + .PHONY: generate_docs generate_docs: @source test/make.sh && generate_docs -# Integration tests - -.PHONY: regional_test_integration -regional_test_integration: - ./test/integration/gcloud/run.sh regional - -.PHONY: zonal_test_integration -zonal_test_integration: - ./test/integration/gcloud/run.sh zonal - -.PHONY: test_integration -test_integration: regional_test_integration zonal_test_integration - @echo "Running tests for regional and zonal clusters" +# Versioning +.PHONY: version +version: + @source helpers/version-repo.sh + +# Build Docker +.PHONY: docker_build_terraform +docker_build_terraform: + docker build -f build/docker/terraform/Dockerfile \ + --build-arg BUILD_TERRAFORM_VERSION=${BUILD_TERRAFORM_VERSION} \ + --build-arg BUILD_CLOUD_SDK_VERSION=${BUILD_CLOUD_SDK_VERSION} \ + --build-arg BUILD_PROVIDER_GOOGLE_VERSION=${BUILD_PROVIDER_GOOGLE_VERSION} \ + --build-arg BUILD_PROVIDER_GSUITE_VERSION=${BUILD_PROVIDER_GSUITE_VERSION} \ + -t ${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM} . + +.PHONY: docker_build_kitchen_terraform +docker_build_kitchen_terraform: + docker build -f build/docker/kitchen_terraform/Dockerfile \ + --build-arg BUILD_TERRAFORM_IMAGE="${DOCKER_IMAGE_TERRAFORM}:${DOCKER_TAG_TERRAFORM}" \ + --build-arg BUILD_RUBY_VERSION="${BUILD_RUBY_VERSION}" \ + -t ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} . + +# Run docker +.PHONY: docker_run +docker_run: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash + +.PHONY: docker_create +docker_create: docker_build_terraform docker_build_kitchen_terraform + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen create" + +.PHONY: docker_converge +docker_converge: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen converge && kitchen converge" + +.PHONY: docker_verify +docker_verify: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen verify" + +.PHONY: docker_destroy +docker_destroy: + docker run --rm -it \ + -v $(CURDIR):/cftk/workdir \ + ${DOCKER_IMAGE_KITCHEN_TERRAFORM}:${DOCKER_TAG_KITCHEN_TERRAFORM} \ + /bin/bash -c "source ${TEST_CONFIG_FILE_LOCATION} && kitchen destroy" + +.PHONY: test_integration_docker +test_integration_docker: docker_create docker_converge docker_verify docker_destroy + @echo "Running test-kitchen tests in docker" + +.PHONY: prepare_test_variables +prepare_test_variables: + @source test/make.sh && prepare_test_variables diff --git a/README.md b/README.md index 1f4bb6d64c..72a1088955 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,6 @@ The project has the following folders and files: ### Requirements - [bundler](https://github.com/bundler/bundler) - [gcloud](https://cloud.google.com/sdk/install) -- [jq](https://stedolan.github.io/jq/) 1.5 - [terraform-docs](https://github.com/segmentio/terraform-docs/releases) 0.3.0 ### Autogeneration of documentation from .tf files @@ -208,12 +207,13 @@ The tests will do the following: - Perform `kitchen validate` command - Performs inspec tests. - Shell out to `gcloud` to validate expected resources in GCP. - - Shell out to `kubectl` to validate expected resource in Kubernetes. - - Shell out to `terraform` to validate outputs. -- Permos `kitchen destroy` command + - Interrogate the cluster to validate expected resource in Kubernetes. +- Perform `kitchen destroy` command - Performs a `terraform destroy -force` -You can use the following command to run the integration test in the root folder +To configure the integration tests, run `make prepare_test_variables` and edit each of the files it outputs to reflect your existing GCP setup. + +You can then use the following command to run the integration test in the root folder `make test_integration` diff --git a/build/docker/kitchen_terraform/Dockerfile b/build/docker/kitchen_terraform/Dockerfile new file mode 100644 index 0000000000..a95f9f8b60 --- /dev/null +++ b/build/docker/kitchen_terraform/Dockerfile @@ -0,0 +1,63 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG BUILD_TERRAFORM_IMAGE +ARG BUILD_RUBY_VERSION +# hadolint ignore=DL3006 +FROM $BUILD_TERRAFORM_IMAGE as cfkt_terraform + + + +FROM ruby:$BUILD_RUBY_VERSION-alpine + +RUN apk add --no-cache \ + bash=4.3.42-r5 \ + curl=7.60.0-r1 \ + git=2.8.6-r0 \ + g++=5.3.0-r0 \ + jq=1.5-r2 \ + make=4.1-r1 \ + musl-dev=1.1.14-r16 \ + python=2.7.14-r0 \ + python-dev=2.7.14-r0 \ + py-pip=8.1.2-r0 + +SHELL ["/bin/bash", "-c"] + +ENV APP_BASE_DIR="/cftk" + +COPY --from=cfkt_terraform $APP_BASE_DIR $APP_BASE_DIR + +ENV HOME="$APP_BASE_DIR/home" +ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH +ENV GOOGLE_APPLICATION_CREDENTIALS="$CREDENTIALS_PATH" \ + CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$CREDENTIALS_PATH" + +# Fix base64 inconsistency +SHELL ["/bin/bash", "-c"] +RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc + +RUN terraform --version && \ + gcloud --version && \ + ruby --version && \ + bundle --version + +COPY ./Gemfile /opt/kitchen/ + +WORKDIR /opt/kitchen +RUN bundle install + +RUN gcloud components install beta --quiet + +WORKDIR $APP_BASE_DIR/workdir diff --git a/build/docker/terraform/Dockerfile b/build/docker/terraform/Dockerfile new file mode 100644 index 0000000000..1d74433a3d --- /dev/null +++ b/build/docker/terraform/Dockerfile @@ -0,0 +1,100 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM alpine:3.8 as builder + +RUN apk add --no-cache \ + bash=4.4.19-r1 \ + git=2.18.0-r0 \ + go=1.10.1-r0 \ + make=4.2.1-r2 \ + musl-dev=1.1.19-r10 + +ENV APP_BASE_DIR="/cftk" + +RUN mkdir -p $APP_BASE_DIR/home && \ + mkdir -p $APP_BASE_DIR/bin && \ + mkdir -p $APP_BASE_DIR/workdir + +ENV GOPATH="/root/go" + +ARG BUILD_PROVIDER_GOOGLE_VERSION +ENV PROVIDER_GOOGLE_VERSION="${BUILD_PROVIDER_GOOGLE_VERSION}" + +RUN mkdir -p $APP_BASE_DIR/home/.terraform.d/plugins && \ + mkdir -p $GOPATH/src/github.com/terraform-providers && \ + git clone https://github.com/terraform-providers/terraform-provider-google.git $GOPATH/src/github.com/terraform-providers/terraform-provider-google + +WORKDIR $GOPATH/src/github.com/terraform-providers/terraform-provider-google +RUN git fetch --all --tags --prune && \ + git checkout tags/v${PROVIDER_GOOGLE_VERSION} -b v${PROVIDER_GOOGLE_VERSION} && \ + make build && \ + mv $GOPATH/bin/terraform-provider-google $APP_BASE_DIR/home/.terraform.d/plugins + + + +FROM alpine:3.8 + +RUN apk add --no-cache \ + bash=4.4.19-r1 \ + curl=7.61.1-r0 \ + git=2.18.0-r0 \ + jq=1.6_rc1-r1 \ + make=4.2.1-r2 \ + python2=2.7.15-r1 + +ENV APP_BASE_DIR="/cftk" + +COPY --from=builder $APP_BASE_DIR $APP_BASE_DIR + +ENV HOME="$APP_BASE_DIR/home" +ENV PATH $APP_BASE_DIR/bin:$APP_BASE_DIR/google-cloud-sdk/bin:$PATH +ENV GOOGLE_APPLICATION_CREDENTIALS="$CREDENTIALS_PATH" \ + CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$CREDENTIALS_PATH" + +# Fix base64 inconsistency +SHELL ["/bin/bash", "-c"] +RUN echo 'base64() { if [[ $@ == "--decode" ]]; then command base64 -d | more; else command base64 "$@"; fi; }' >> $APP_BASE_DIR/home/.bashrc + +ARG BUILD_CLOUD_SDK_VERSION +ENV CLOUD_SDK_VERSION="${BUILD_CLOUD_SDK_VERSION}" + +WORKDIR $APP_BASE_DIR +RUN curl -LO https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + tar xzf google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + rm google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz && \ + ln -s /lib /lib64 && \ + gcloud config set core/disable_usage_reporting true && \ + gcloud config set component_manager/disable_update_check true && \ + gcloud config set metrics/environment github_docker_image && \ + gcloud --version + +ARG BUILD_TERRAFORM_VERSION +ENV TERRAFORM_VERSION="${BUILD_TERRAFORM_VERSION}" + +RUN curl -LO https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + rm terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ + mv terraform $APP_BASE_DIR/bin && \ + terraform --version + +ARG BUILD_PROVIDER_GSUITE_VERSION +ENV PROVIDER_GSUITE_VERSION="${BUILD_PROVIDER_GSUITE_VERSION}" + +RUN curl -LO https://github.com/DeviaVir/terraform-provider-gsuite/releases/download/v${PROVIDER_GSUITE_VERSION}/terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + tar xzf terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + rm terraform-provider-gsuite_${PROVIDER_GSUITE_VERSION}_linux_amd64.tgz && \ + mv terraform-provider-gsuite_v${PROVIDER_GSUITE_VERSION} $APP_BASE_DIR/home/.terraform.d/plugins/terraform-provider-gsuite + +WORKDIR $APP_BASE_DIR/workdir diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md index fa52c2b491..b1db8c2815 100644 --- a/examples/deploy_service/README.md +++ b/examples/deploy_service/README.md @@ -8,16 +8,43 @@ It will: - Create an Nginx Pod - Create an Nginx Service -Expected variables: -- `project_id` -- `region` -- `network` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| master_kubernetes_version | The master Kubernetes version | +| network | | +| project_id | | +| region | | +| region_example | | +| subnetwork | | + +[^]: (autogen_docs_end) To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index 8624acdf26..2dd7b2489d 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -14,12 +14,9 @@ * limitations under the License. */ -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - provider "google" { - credentials = "${file(local.credentials_file_path)}" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" } provider "kubernetes" { diff --git a/examples/deploy_service/outputs.tf b/examples/deploy_service/outputs.tf index 17dff21946..681e32e5c3 100644 --- a/examples/deploy_service/outputs.tf +++ b/examples/deploy_service/outputs.tf @@ -14,23 +14,61 @@ * limitations under the License. */ -output "name_example" { +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { description = "Cluster name" value = "${module.gke.name}" } -output "endpoint_example" { - sensitive = true - description = "Cluster endpoint" - value = "${module.gke.endpoint}" +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "region_example" { + value = "${module.gke.region}" +} + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + sensitive = true + value = "${module.gke.ca_certificate}" +} + +output "location" { + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" } -output "location_example" { - description = "Cluster location" - value = "${module.gke.location}" +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" } -output "zones_example" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" } diff --git a/examples/deploy_service/variables.tf b/examples/deploy_service/variables.tf index f49f87a61c..dc111ef220 100644 --- a/examples/deploy_service/variables.tf +++ b/examples/deploy_service/variables.tf @@ -18,6 +18,10 @@ variable "project_id" { description = "The project ID to host the cluster in" } +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + variable "region" { description = "The region to host the cluster in" } diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index cc8972d0b0..a96a3c619d 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -2,17 +2,43 @@ This example illustrates how to create a cluster with multiple custom node-pool configurations with node labels, taints, and network tags. -Expected variables: -- `project_id` -- `region` -- `network` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` -- `pool_01_service_account` - Only needed if you've deleted the default service account from your project +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| pool_01_service_account | Service account to associate to the nodes on pool-01. Only needed if you have deleted or otherwise don't want to use the default compute service account | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| network | | +| project_id | | +| region | | +| region_example | | +| subnetwork | | + +[^]: (autogen_docs_end) To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index cac946b6a9..7140a2efa8 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -14,12 +14,9 @@ * limitations under the License. */ -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - provider "google" { - credentials = "${file(local.credentials_file_path)}" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" } module "gke" { @@ -35,12 +32,12 @@ module "gke" { node_pools = [ { name = "pool-01" - min_count = 4 + min_count = 2 }, { name = "pool-02" machine_type = "n1-standard-2" - min_count = 2 + min_count = 1 max_count = 3 disk_size_gb = 30 disk_type = "pd-standard" @@ -95,3 +92,5 @@ module "gke" { pool-02 = [] } } + +data "google_client_config" "default" {} diff --git a/examples/node_pool/outputs.tf b/examples/node_pool/outputs.tf index 6ab4d1fe82..9c4440771a 100644 --- a/examples/node_pool/outputs.tf +++ b/examples/node_pool/outputs.tf @@ -14,31 +14,56 @@ * limitations under the License. */ -output "name_example" { +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { description = "Cluster name" value = "${module.gke.name}" } -output "endpoint_example" { - sensitive = true - description = "Cluster endpoint" - value = "${module.gke.endpoint}" +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "region_example" { + value = "${module.gke.region}" +} + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" } -output "location_example" { - description = "Cluster location" - value = "${module.gke.location}" +output "ca_certificate" { + sensitive = true + value = "${module.gke.ca_certificate}" } -output "zones_example" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" +output "location" { + value = "${module.gke.location}" } -output "node_pools_names_example" { - value = "${module.gke.node_pools_names}" +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" } -output "node_pools_versions_example" { - value = "${module.gke.node_pools_versions}" +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" } diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 0ef3e3be2a..118b93160b 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -18,6 +18,10 @@ variable "project_id" { description = "The project ID to host the cluster in" } +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + variable "region" { description = "The region to host the cluster in" } @@ -39,5 +43,5 @@ variable "ip_range_services" { } variable "pool_01_service_account" { - description = "Service account to associate to the nodes on pool-01" + description = "Service account to associate to the nodes on pool-01. Only needed if you have deleted or otherwise don't want to use the default compute service account" } diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md deleted file mode 100644 index c882008564..0000000000 --- a/examples/shared_vpc/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Shared VPC Cluster - -This example illustrates how to create a simple cluster. - -Expected variables: -- `project_id` -- `region` -- `network` -- `network_project_id` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` - -To provision this example, run the following from within this directory: -- `terraform init` to get the plugins -- `terraform plan` to see the infrastructure plan -- `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure diff --git a/examples/shared_vpc/main.tf b/examples/shared_vpc/main.tf deleted file mode 100644 index c68a652100..0000000000 --- a/examples/shared_vpc/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - -provider "google" { - credentials = "${file(local.credentials_file_path)}" -} - -module "gke" { - source = "../../" - project_id = "${var.project_id}" - name = "shared-vpc-sample-cluster" - region = "${var.region}" - network = "${var.network}" - network_project_id = "${var.network_project_id}" - subnetwork = "${var.subnetwork}" - ip_range_pods = "${var.ip_range_pods}" - ip_range_services = "${var.ip_range_services}" -} diff --git a/examples/shared_vpc/outputs.tf b/examples/shared_vpc/outputs.tf deleted file mode 100644 index 17dff21946..0000000000 --- a/examples/shared_vpc/outputs.tf +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "name_example" { - description = "Cluster name" - value = "${module.gke.name}" -} - -output "endpoint_example" { - sensitive = true - description = "Cluster endpoint" - value = "${module.gke.endpoint}" -} - -output "location_example" { - description = "Cluster location" - value = "${module.gke.location}" -} - -output "zones_example" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" -} diff --git a/examples/shared_vpc/variables.tf b/examples/shared_vpc/variables.tf deleted file mode 100644 index 8ac12e4f45..0000000000 --- a/examples/shared_vpc/variables.tf +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The project ID to host the cluster in" -} - -variable "region" { - description = "The region to host the cluster in" -} - -variable "network" { - description = "The VPC network to host the cluster in" -} - -variable "subnetwork" { - description = "The subnetwork to host the cluster in" -} - -variable "network_project_id" { - description = "The project ID of the shared VPC's host" -} - -variable "ip_range_pods" { - description = "The secondary ip range to use for pods" -} - -variable "ip_range_services" { - description = "The secondary ip range to use for pods" -} diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md index dccf8dcc93..c0d60db5d5 100644 --- a/examples/simple_regional/README.md +++ b/examples/simple_regional/README.md @@ -2,16 +2,41 @@ This example illustrates how to create a simple cluster. -Expected variables: -- `project_id` -- `region` -- `network` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | Cluster endpoint | +| location | Cluster location | +| master_kubernetes_version | The master Kubernetes version | +| network | Network the cluster is provisioned in | +| project_id | | +| region | | +| subnetwork | Subnetwork the cluster is provisioned in | + +[^]: (autogen_docs_end) To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 72a9581f78..40cc858640 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -14,12 +14,9 @@ * limitations under the License. */ -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - provider "google" { - credentials = "${file(local.credentials_file_path)}" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" } module "gke" { @@ -33,3 +30,5 @@ module "gke" { ip_range_pods = "${var.ip_range_pods}" ip_range_services = "${var.ip_range_services}" } + +data "google_client_config" "default" {} diff --git a/examples/simple_regional/outputs.tf b/examples/simple_regional/outputs.tf index d7ff58179e..dd74cf3ca5 100644 --- a/examples/simple_regional/outputs.tf +++ b/examples/simple_regional/outputs.tf @@ -14,18 +14,56 @@ * limitations under the License. */ -output "name_example" { +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { description = "Cluster name" value = "${module.gke.name}" } -output "endpoint_example" { +output "network" { + description = "Network the cluster is provisioned in" + value = "${var.network}" +} + +output "subnetwork" { + description = "Subnetwork the cluster is provisioned in" + value = "${var.subnetwork}" +} + +output "kubernetes_endpoint" { sensitive = true description = "Cluster endpoint" value = "${module.gke.endpoint}" } -output "location_example" { +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "location" { description = "Cluster location" value = "${module.gke.location}" } + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/simple_regional/variables.tf b/examples/simple_regional/variables.tf index f49f87a61c..dc111ef220 100644 --- a/examples/simple_regional/variables.tf +++ b/examples/simple_regional/variables.tf @@ -18,6 +18,10 @@ variable "project_id" { description = "The project ID to host the cluster in" } +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + variable "region" { description = "The region to host the cluster in" } diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md index a6de591759..0b66ab1480 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal/README.md @@ -2,16 +2,42 @@ This example illustrates how to create a simple cluster. -Expected variables: -- `project_id` -- `region` -- `network` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | Cluster endpoint | +| location | Cluster location | +| master_kubernetes_version | The master Kubernetes version | +| network | Network the cluster is provisioned in | +| project_id | | +| region | | +| subnetwork | Subnetwork the cluster is provisioned in | + +[^]: (autogen_docs_end) To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf index 7ec5f5a0ea..3c8d918fb4 100644 --- a/examples/simple_zonal/main.tf +++ b/examples/simple_zonal/main.tf @@ -14,12 +14,8 @@ * limitations under the License. */ -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - provider "google" { - credentials = "${file(local.credentials_file_path)}" + credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -35,3 +31,5 @@ module "gke" { ip_range_pods = "${var.ip_range_pods}" ip_range_services = "${var.ip_range_services}" } + +data "google_client_config" "default" {} diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf index 17dff21946..dd74cf3ca5 100644 --- a/examples/simple_zonal/outputs.tf +++ b/examples/simple_zonal/outputs.tf @@ -14,23 +14,56 @@ * limitations under the License. */ -output "name_example" { +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { description = "Cluster name" value = "${module.gke.name}" } -output "endpoint_example" { +output "network" { + description = "Network the cluster is provisioned in" + value = "${var.network}" +} + +output "subnetwork" { + description = "Subnetwork the cluster is provisioned in" + value = "${var.subnetwork}" +} + +output "kubernetes_endpoint" { sensitive = true description = "Cluster endpoint" value = "${module.gke.endpoint}" } -output "location_example" { +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "location" { description = "Cluster location" value = "${module.gke.location}" } -output "zones_example" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" } diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal/variables.tf index 3d5877911a..fb4755571d 100644 --- a/examples/simple_zonal/variables.tf +++ b/examples/simple_zonal/variables.tf @@ -18,6 +18,10 @@ variable "project_id" { description = "The project ID to host the cluster in" } +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + variable "region" { description = "The region to host the cluster in" } diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md index d47372411b..638b696a82 100644 --- a/examples/stub_domains/README.md +++ b/examples/stub_domains/README.md @@ -7,16 +7,43 @@ It will: - Remove the default kube-dns configmap - Add a new kube-dns configmap with custom stub domains -Expected variables: -- `project_id` -- `region` -- `network` -- `subnetwork` -- `ip_range_pods` -- `ip_range_services` +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca_certificate | | +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | | +| location | | +| network | | +| project_id | | +| region | | +| region_example | | +| subnetwork | | + +[^]: (autogen_docs_end) To provision this example, run the following from within this directory: - `terraform init` to get the plugins - `terraform plan` to see the infrastructure plan - `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index e89a68396f..55e50b98d5 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -14,12 +14,9 @@ * limitations under the License. */ -locals { - credentials_file_path = "${path.module}/sa-key.json" -} - provider "google" { - credentials = "${file(local.credentials_file_path)}" + credentials = "${file(var.credentials_path)}" + region = "${var.region}" } module "gke" { @@ -31,6 +28,7 @@ module "gke" { subnetwork = "${var.subnetwork}" ip_range_pods = "${var.ip_range_pods}" ip_range_services = "${var.ip_range_services}" + network_policy = true stub_domains { "example.com" = [ @@ -44,3 +42,5 @@ module "gke" { ] } } + +data "google_client_config" "default" {} diff --git a/examples/stub_domains/outputs.tf b/examples/stub_domains/outputs.tf index 17dff21946..9c4440771a 100644 --- a/examples/stub_domains/outputs.tf +++ b/examples/stub_domains/outputs.tf @@ -14,23 +14,56 @@ * limitations under the License. */ -output "name_example" { +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { description = "Cluster name" value = "${module.gke.name}" } -output "endpoint_example" { - sensitive = true - description = "Cluster endpoint" - value = "${module.gke.endpoint}" +output "network" { + value = "${var.network}" +} + +output "subnetwork" { + value = "${var.subnetwork}" +} + +output "region_example" { + value = "${module.gke.region}" +} + +output "kubernetes_endpoint" { + sensitive = true + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "ca_certificate" { + sensitive = true + value = "${module.gke.ca_certificate}" +} + +output "location" { + value = "${module.gke.location}" } -output "location_example" { - description = "Cluster location" - value = "${module.gke.location}" +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" } -output "zones_example" { - description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" } diff --git a/examples/stub_domains/variables.tf b/examples/stub_domains/variables.tf index f49f87a61c..fb4755571d 100644 --- a/examples/stub_domains/variables.tf +++ b/examples/stub_domains/variables.tf @@ -18,10 +18,19 @@ variable "project_id" { description = "The project ID to host the cluster in" } +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + variable "region" { description = "The region to host the cluster in" } +variable "zones" { + type = "list" + description = "The zone to host the cluster in (required if is a zonal cluster)" +} + variable "network" { description = "The VPC network to host the cluster in" } diff --git a/helpers/combine_docfiles.py b/helpers/combine_docfiles.py index b01af7c571..16516cb009 100644 --- a/helpers/combine_docfiles.py +++ b/helpers/combine_docfiles.py @@ -25,20 +25,28 @@ regex specified here ''' +import os import re import sys -insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' -exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' +insert_separator_regex = '(.*?\[\^\]\:\ \(autogen_docs_start\))(.*?)(\n\[\^\]\:\ \(autogen_docs_end\).*?$)' # noqa: E501 +exclude_separator_regex = '(.*?)Copyright 20\d\d Google LLC.*?limitations under the License.(.*?)$' # noqa: E501 if len(sys.argv) != 3: - sys.exit(1) + sys.exit(1) + +if not os.path.isfile(sys.argv[1]): + sys.exit(0) input = open(sys.argv[1], "r").read() replace_content = open(sys.argv[2], "r").read() # Exclude the specified content from the replacement content -groups = re.match(exclude_separator_regex, replace_content, re.DOTALL).groups(0) +groups = re.match( + exclude_separator_regex, + replace_content, + re.DOTALL +).groups(0) replace_content = groups[0] + groups[1] # Find where to put the replacement content, overwrite the input file diff --git a/main.tf b/main.tf index 74b93940fe..c9107d18ec 100644 --- a/main.tf +++ b/main.tf @@ -23,7 +23,7 @@ data "google_compute_zones" "available" { } resource "random_shuffle" "available_zones" { - input = ["${data.google_compute_zones.available.names}"] + input = ["${data.google_compute_zones.available.names}"] result_count = 3 } diff --git a/networks.tf b/networks.tf index eecdac2d94..47fee1bec3 100644 --- a/networks.tf +++ b/networks.tf @@ -1,3 +1,18 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ data "google_compute_network" "gke_network" { name = "${var.network}" project = "${local.network_project_id}" diff --git a/test/integration/gcloud/.kitchen.yml b/test/fixtures/config.sh old mode 100644 new mode 100755 similarity index 74% rename from test/integration/gcloud/.kitchen.yml rename to test/fixtures/config.sh index 65885a1b7b..2efbdb4f57 --- a/test/integration/gcloud/.kitchen.yml +++ b/test/fixtures/config.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,22 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- -driver: - name: "terraform" - command_timeout: 1800 - -provisioner: - name: "terraform" - -transport: - name: exec - -platforms: - - name: local - -verifier: - name: inspec - -suites: - - name: "default" +CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="$(pwd)/credentials.json" +export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE diff --git a/test/fixtures/deploy_service/terraform.tfvars.sample b/test/fixtures/deploy_service/terraform.tfvars.sample new file mode 100644 index 0000000000..ba9b99be4c --- /dev/null +++ b/test/fixtures/deploy_service/terraform.tfvars.sample @@ -0,0 +1,8 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" + diff --git a/test/fixtures/node_pool/terraform.tfvars.sample b/test/fixtures/node_pool/terraform.tfvars.sample new file mode 100644 index 0000000000..d1bd27aa64 --- /dev/null +++ b/test/fixtures/node_pool/terraform.tfvars.sample @@ -0,0 +1,8 @@ +project_id="" +credentials_path="../../../credentials.json" +region="us-east4" +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" +pool_01_service_account="" diff --git a/test/fixtures/simple_regional/terraform.tfvars.sample b/test/fixtures/simple_regional/terraform.tfvars.sample new file mode 100644 index 0000000000..e17012a071 --- /dev/null +++ b/test/fixtures/simple_regional/terraform.tfvars.sample @@ -0,0 +1,7 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" diff --git a/test/fixtures/simple_zonal/terraform.tfvars.sample b/test/fixtures/simple_zonal/terraform.tfvars.sample new file mode 100644 index 0000000000..d469c8925f --- /dev/null +++ b/test/fixtures/simple_zonal/terraform.tfvars.sample @@ -0,0 +1,8 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +zones=["us-east4-b","us-east4-c","us-east4-a"] +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" diff --git a/test/fixtures/stub_domains/terraform.tfvars.sample b/test/fixtures/stub_domains/terraform.tfvars.sample new file mode 100644 index 0000000000..8b38e764ce --- /dev/null +++ b/test/fixtures/stub_domains/terraform.tfvars.sample @@ -0,0 +1,9 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +zones=["us-east4-b","us-east4-c","us-east4-a"] +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" + diff --git a/test/integration/deploy_service/controls/gcloud.rb b/test/integration/deploy_service/controls/gcloud.rb new file mode 100644 index 0000000000..ce1568f18e --- /dev/null +++ b/test/integration/deploy_service/controls/gcloud.rb @@ -0,0 +1,64 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') +master_kubernetes_version = attribute('master_kubernetes_version') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "has the expected initial cluster version" do + expect(data['initialClusterVersion']).to eq master_kubernetes_version + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + end + end +end diff --git a/test/integration/deploy_service/controls/kubectl.rb b/test/integration/deploy_service/controls/kubectl.rb new file mode 100644 index 0000000000..1443f94057 --- /dev/null +++ b/test/integration/deploy_service/controls/kubectl.rb @@ -0,0 +1,67 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'kubeclient' +require 'rest-client' + +require 'base64' + +kubernetes_endpoint = attribute('kubernetes_endpoint') +client_token = attribute('client_token') +ca_certificate = attribute('ca_certificate') + +control "kubectl" do + title "Kubernetes configuration" + + describe "kubernetes" do + let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } + let(:client) do + cert_store = OpenSSL::X509::Store.new + cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) + Kubeclient::Client.new( + kubernetes_http_endpoint, + "v1", + ssl_options: { + cert_store: cert_store, + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + }, + auth_options: { + bearer_token: Base64.decode64(client_token), + }, + ) + end + + describe "services" do + describe "nginx" do + let(:service) { client.get_service("terraform-example", "default") } + let(:service_load_balancer_ip) { service.status.loadBalancer.ingress.first.ip } + let(:service_load_balancer_address) { "http://#{service_load_balancer_ip}:8080" } + + it "exists" do + expect(service).not_to be_nil + end + + it "has an IP address" do + expect(service_load_balancer_ip).not_to be_nil + end + + it "is reachable" do + expect { + RestClient.get(service_load_balancer_address) + }.to_not raise_exception + end + end + end + end +end diff --git a/test/integration/deploy_service/inspec.yml b/test/integration/deploy_service/inspec.yml new file mode 100644 index 0000000000..b4748afdf9 --- /dev/null +++ b/test/integration/deploy_service/inspec.yml @@ -0,0 +1,40 @@ +name: deploy_service +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: ca_certificate + required: true + type: string diff --git a/test/integration/gcloud/.gitignore b/test/integration/gcloud/.gitignore deleted file mode 100644 index cdd7c19c8d..0000000000 --- a/test/integration/gcloud/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config.sh diff --git a/test/integration/gcloud/run.sh b/test/integration/gcloud/run.sh deleted file mode 100755 index ba0df4aea1..0000000000 --- a/test/integration/gcloud/run.sh +++ /dev/null @@ -1,352 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if [ "$#" -lt 1 ]; then - >&2 echo "Must specify cluster type (regional/zonal)" - exit 1 -fi - -export CLUSTER_TYPE="$1" - -TEMPDIR=$(pwd)/test/integration/tmp -TESTDIR=${BASH_SOURCE%/*} - -function export_vars() { - export TEST_ID="modules_gke_integration_gcloud_${RANDOM}" - export KUBECONFIG="${TEMPDIR}/${CLUSTER_TYPE}/${TEST_ID}.kubeconfig" - if [[ $CLUSTER_TYPE = "regional" ]]; then - if [ -f "./regional_config.sh" ]; then - source ./regional_config.sh - fi - export CLUSTER_REGIONAL="true" - export CLUSTER_LOCATION="$REGIONAL_LOCATION" - export CLUSTER_NAME="$REGIONAL_CLUSTER_NAME" - export IP_RANGE_PODS="$REGIONAL_IP_RANGE_PODS" - export IP_RANGE_SERVICES="$REGIONAL_IP_RANGE_SERVICES" - else - if [ -f "./zonal_config.sh" ]; then - source ./zonal_config.sh - fi - if [ -z "${ZONE}" ]; then - echo "Can not create a zonal cluster without specifying \$ZONE. Aborting..." - exit 1 - fi - export CLUSTER_REGIONAL="false" - export CLUSTER_LOCATION="$ZONAL_LOCATION" - export CLUSTER_NAME="$ZONAL_CLUSTER_NAME" - export IP_RANGE_PODS="$ZONAL_IP_RANGE_PODS" - export IP_RANGE_SERVICES="$ZONAL_IP_RANGE_SERVICES" - fi - - if [ "${ZONE}" = "" ] && [ "${ADDITIONAL_ZONES}" = "" ]; then - export ZONES="" - else - export ZONES="\"$ZONE\",$ADDITIONAL_ZONES" - fi -} - -# Activate test working directory -function make_testdir() { - mkdir -p "${TEMPDIR}/${CLUSTER_TYPE}" - cp -r "${TESTDIR}"/* "${TEMPDIR}/${CLUSTER_TYPE}/" - cp -r "$TESTDIR"/.kitchen.yml "${TEMPDIR}/${CLUSTER_TYPE}/" -} - -# Activate test config -function activate_config() { - # shellcheck disable=SC1091 - source config.sh - echo "$PROJECT_NAME" -} - -# Cleans the workdir -function clean_workdir() { - #rm -rf "$TEMPDIR" - - export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="" - unset CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE - -} - -# Creates the main.tf file for Terraform -function create_main_tf_file() { - echo "Creating main.tf file" - cat < main.tf -locals { - credentials_file_path = "$CREDENTIALS_PATH" -} - -provider "google" { - credentials = "\${file(local.credentials_file_path)}" - region = "${REGION}" -} - -provider "kubernetes" { - load_config_file = false - host = "https://\${module.gke.endpoint}" - token = "\${data.google_client_config.default.access_token}" - cluster_ca_certificate = "\${base64decode(module.gke.ca_certificate)}" -} - -data "google_client_config" "default" {} - -module "gke" { - source = "../../../../" - project_id = "$PROJECT_ID" - name = "$CLUSTER_NAME" - description = "Test GKE cluster" - regional = $CLUSTER_REGIONAL - region = "$REGION" - zones = [$ZONES] - kubernetes_version = "$KUBERNETES_VERSION" - network = "$NETWORK" - subnetwork = "$SUBNETWORK" - ip_range_pods = "$IP_RANGE_PODS" - ip_range_services = "$IP_RANGE_SERVICES" - - http_load_balancing = false - horizontal_pod_autoscaling = true - kubernetes_dashboard = true - network_policy = true - - stub_domains { - "example.com" = [ - "10.254.154.11", - "10.254.154.12", - ] - - "testola.com" = [ - "10.254.154.11", - "10.254.154.12", - ] - } - - non_masquerade_cidrs = [ - "10.0.0.0/8", - "192.168.20.0/24", - "192.168.21.0/24", - ] - - node_pools = [ - { - name = "pool-01" - machine_type = "n1-standard-1" - image_type = "COS" - initial_node_count = 2 - min_count = 1 - max_count = 2 - auto_upgrade = false - disk_size_gb = 30 - disk_type = "pd-standard" - service_account = "$NODE_POOL_SERVICE_ACCOUNT" - }, - ] - node_pools_labels = { - all = { - all_pools_label = "something" - } - - pool-01 = { - pool_01_label = "yes" - pool_01_another_label = "no" - } - } - node_pools_taints = { - all = [ - { - key = "all_pools_taint" - value = "true" - effect = "PREFER_NO_SCHEDULE" - }, - ] - - pool-01 = [ - { - key = "pool_01_taint" - value = "true" - effect = "PREFER_NO_SCHEDULE" - }, - { - key = "pool_01_another_taint" - value = "true" - effect = "PREFER_NO_SCHEDULE" - }, - ] - } - node_pools_tags = { - all = [ - "all-node-network-tag", - ] - - pool-01 = [ - "pool-01-network-tag", - ] - } -} - -resource "kubernetes_pod" "nginx-example" { - metadata { - name = "nginx-example" - - labels { - maintained_by = "terraform" - app = "nginx-example" - } - } - - spec { - container { - image = "nginx:1.7.9" - name = "nginx-example" - } - } - - depends_on = ["module.gke"] -} - -resource "kubernetes_service" "nginx-example" { - metadata { - name = "terraform-example" - } - - spec { - selector { - app = "\${kubernetes_pod.nginx-example.metadata.0.labels.app}" - } - - session_affinity = "ClientIP" - - port { - port = 8080 - target_port = 80 - } - - type = "LoadBalancer" - } - - depends_on = ["module.gke"] -} - -EOF -} - -# Creates the outputs.tf file -function create_outputs_file() { - echo "Creating outputs.tf file" - cat <<'EOF' > outputs.tf -output "name_example" { - value = "${module.gke.name}" -} - -output "type_example" { - value = "${module.gke.type}" -} - -output "location_example" { - value = "${module.gke.location}" -} - -output "region_example" { - value = "${module.gke.region}" -} - -output "zones_example" { - value = "${module.gke.zones}" -} - -output "endpoint_example" { - sensitive = true - value = "${module.gke.endpoint}" -} - -output "ca_certificate_example" { - sensitive = true - value = "${module.gke.ca_certificate}" -} - -output "min_master_version_example" { - value = "${module.gke.min_master_version}" -} - -output "master_version_example" { - value = "${module.gke.master_version}" -} - -output "network_policy_example" { - value = "${module.gke.network_policy_enabled}" -} - -output "http_load_balancing_example" { - value = "${module.gke.http_load_balancing_enabled}" -} - -output "horizontal_pod_autoscaling_example" { - value = "${module.gke.horizontal_pod_autoscaling_enabled}" -} - -output "kubernetes_dashboard_example" { - value = "${module.gke.kubernetes_dashboard_enabled}" -} - -output "node_pools_names_example" { - value = "${module.gke.node_pools_names}" -} - -output "node_pools_versions_example" { - value = "${module.gke.node_pools_versions}" -} - -# For use in integration tests -output "module_path" { - value = "${path.module}/../../../../" -} - -output "client_token" { - sensitive = true - value = "${base64encode(data.google_client_config.default.access_token)}" -} - -EOF -} - -# Install gems -function bundle_install() { - bundle install -} - -# Execute kitchen tests -function run_kitchen() { - bundle exec kitchen create - bundle exec kitchen converge - bundle exec kitchen converge # second time to enable network policy - bundle exec kitchen verify - bundle exec kitchen destroy -} - -# Preparing environment -make_testdir - -cd "${TEMPDIR}/${CLUSTER_TYPE}/" || exit -activate_config -export_vars zonal -create_main_tf_file -create_outputs_file -bundle_install -run_kitchen - -# # # Clean the environment -cd - || exit -clean_workdir -echo "Integration test finished" diff --git a/test/integration/gcloud/sample.sh b/test/integration/gcloud/sample.sh deleted file mode 100644 index 5c6cc1f6a8..0000000000 --- a/test/integration/gcloud/sample.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -################################################################# -# PLEASE FILL THE VARIABLES WITH VALID VALUES FOR TESTING # -# DO NOT REMOVE ANY OF THE VARIABLES # -################################################################# - -## These values you *MUST* modify to match your environment -export PROJECT_ID="gke-test-integration" -export CREDENTIALS_PATH="$HOME/sa-key.json" -export NETWORK="vpc-01" -export SUBNETWORK="us-east4-01" -export REGIONAL_IP_RANGE_PODS="us-east4-01-gke-01-pod" -export REGIONAL_IP_RANGE_SERVICES="us-east4-01-gke-01-service" -export ZONAL_IP_RANGE_PODS="us-east4-01-gke-02-pod" -export ZONAL_IP_RANGE_SERVICES="us-east4-01-gke-02-service" - -## These values you can potentially leave at the defaults -export CLUSTER_NAME="int-test-cluster-01" -export REGION="us-east4" -export ZONE="us-east4-a" -export ADDITIONAL_ZONES='"us-east4-b","us-east4-c"' -export KUBERNETES_VERSION="1.10.6-gke.2" -export NODE_POOL_SERVICE_ACCOUNT="" -export REGIONAL_CLUSTER_NAME="int-test-regional-01" -export REGIONAL_LOCATION="$REGION" -export ZONAL_CLUSTER_NAME="int-test-zonal-01" -export ZONAL_LOCATION="$ZONE" -export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$CREDENTIALS_PATH -export GOOGLE_APPLICATION_CREDENTIALS=$CREDENTIALS_PATH diff --git a/test/integration/gcloud/test/integration/default/inspec/gcloud.rb b/test/integration/gcloud/test/integration/default/inspec/gcloud.rb deleted file mode 100644 index b88743cb7c..0000000000 --- a/test/integration/gcloud/test/integration/default/inspec/gcloud.rb +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test the cluster is in running status -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json| jq -cre \'.status\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'RUNNING' } -end - -# Test the cluster has the expected initial cluster version -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.initialClusterVersion\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['KUBERNETES_VERSION'] } -end - -# Test the cluster is in the expected network -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.network\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['NETWORK'] } -end - -# Test the cluster is in the expected subnetwork -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.subnetwork\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['SUBNETWORK'] } -end - -# Test the cluster has the expected secondary ip range for pods -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.ipAllocationPolicy.clusterSecondaryRangeName\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['IP_RANGE_PODS'] } -end - -# Test the cluster has the expected secondary ip range for services -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.ipAllocationPolicy.servicesSecondaryRangeName\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['IP_RANGE_SERVICES'] } -end - -# Test the cluster has the expected addon settings -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.addonsConfig\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '{"horizontalPodAutoscaling":{},"httpLoadBalancing":{"disabled":true},"kubernetesDashboard":{},"networkPolicyConfig":{}}' } -end - -# Test default pool has no initial node count -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "default-pool") | .initialNodeCount\'') do - its('exit_status') { should eq 1 } - its('stdout.strip') { should eq 'null' } -end - -# Test default pool has not auto scaling enabled -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "default-pool") | .autoscaling.enabled\'') do - its('exit_status') { should eq 1 } - its('stdout.strip') { should eq 'null' } -end - -# Test pool-01 is expected version -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .version\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq ENV['KUBERNETES_VERSION'] } -end - -# Test pool-01 has auto scaling enabled -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .autoscaling.enabled\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'true' } -end - -# Test pool-01 has expected min node count -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .autoscaling.minNodeCount\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '1' } -end - -# Test pool-01 has expected max node count -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .autoscaling.maxNodeCount\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '2' } -end - -# Test pool-01 is expected machine type -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .config.machineType\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'n1-standard-1' } -end - -# Test pool-01 has expected disk size -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .config.diskSizeGb\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '30' } -end - -# Test pool-01 has expected labels -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .config.labels\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '{"all_pools_label":"something","cluster_name":"' + ENV['CLUSTER_NAME'] + '","node_pool":"pool-01","pool_01_another_label":"no","pool_01_label":"yes"}' } -end - -# Test pool-01 has expected network tags -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .config.tags\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '["gke-' + ENV['CLUSTER_NAME'] + '","gke-' + ENV['CLUSTER_NAME'] + '-pool-01","all-node-network-tag","pool-01-network-tag"]' } -end - -# Test pool-01 has auto repair enabled -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .management.autoRepair\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'true' } -end - -# Test pool-01 has auto upgrade disabled -describe command('gcloud --project=${PROJECT_ID} container clusters --zone=${CLUSTER_LOCATION} describe ${CLUSTER_NAME} --format=json | jq -cre \'.nodePools[] | select(.name == "pool-01") | .management.autoUpgrade\'') do - its('exit_status') { should eq 1 } - its('stdout.strip') { should eq 'null' } -end diff --git a/test/integration/gcloud/test/integration/default/inspec/kubectl.rb b/test/integration/gcloud/test/integration/default/inspec/kubectl.rb deleted file mode 100644 index ed4e077b70..0000000000 --- a/test/integration/gcloud/test/integration/default/inspec/kubectl.rb +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Test pool-01 has expected taints -describe command('$(terraform output module_path)/scripts/kubectl_wrapper.sh https://$(terraform output endpoint_example) $(terraform output client_token | base64 --decode) $(terraform output ca_certificate_example) kubectl get nodes -o json -l node_pool=pool-01 | jq -cre \'.items[0].spec.taints\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq '[{"effect":"PreferNoSchedule","key":"all_pools_taint","value":"true"},{"effect":"PreferNoSchedule","key":"pool_01_taint","value":"true"},{"effect":"PreferNoSchedule","key":"pool_01_another_taint","value":"true"}]' } -end - -# Test kube dns configmap created" { -describe command('$(terraform output module_path)/scripts/kubectl_wrapper.sh https://$(terraform output endpoint_example) $(terraform output client_token | base64 --decode) $(terraform output ca_certificate_example) kubectl -n kube-system get configmap -o json kube-dns | jq -cre \'.metadata.labels.maintained_by\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'terraform' } -end - -# Test ip masq agent configmap created" { -describe command('$(terraform output module_path)/scripts/kubectl_wrapper.sh https://$(terraform output endpoint_example) $(terraform output client_token | base64 --decode) $(terraform output ca_certificate_example) kubectl -n kube-system get configmap -o json ip-masq-agent | jq -cre \'.metadata.labels.maintained_by\'') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should eq 'terraform' } -end - -# Test that the nginx example service is reachable" { -describe command('curl -Ifs -m 10 $($(terraform output module_path)/scripts/kubectl_wrapper.sh https://$(terraform output endpoint_example) $(terraform output client_token | base64 --decode) $(terraform output ca_certificate_example) kubectl get service terraform-example -o json | jq -cre \'.status.loadBalancer.ingress[0].ip\'):8080') do - its('exit_status') { should eq 0 } - its('stdout.strip') { should include 'HTTP/1.1 200 OK' } - its('stdout.strip') { should include 'Server: nginx' } -end diff --git a/test/integration/gcloud/test/integration/default/inspec/terraform.rb b/test/integration/gcloud/test/integration/default/inspec/terraform.rb deleted file mode 100644 index 71a4bcc313..0000000000 --- a/test/integration/gcloud/test/integration/default/inspec/terraform.rb +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require_relative '../../../../test/support/google_cloud.rb' - -# Test the name output -describe command('terraform output name_example') do - its('stdout.strip') { should eq ENV['CLUSTER_NAME'] } -end - -# Test the location output -describe command('terraform output type_example') do - its('stdout.strip') { should eq ENV['CLUSTER_TYPE'] } -end - -# Test the location output -describe command('terraform output location_example') do - its('stdout.strip') { should eq ENV['CLUSTER_LOCATION'] } -end - -# Test the region output -describe command('terraform output region_example') do - its('stdout.strip') { should eq ENV['REGION'] } -end - -# Test the zones output -describe command('terraform output -json zones_example | jq -cre \'.value\'') do - if ENV['ZONES'] != '' - its('stdout.strip') { should eq '[' + ENV['ZONES'] + ']' } - else - it "should be 3 zones in the region" do - zones = JSON.parse(subject.stdout.strip) - zones.count.should be 3 - - available_zones = google_compute_service.get_region(ENV['PROJECT_ID'], ENV['REGION']).zones.map { |z| z.split("/").last } - zones.each do |z| - available_zones.should include z - end - end - end -end - -# Test the endpoint output -describe command('terraform output endpoint_example') do - its('stdout.strip') { should_not eq '' } -end - -# Test the ca_certificate output -describe command('terraform output ca_certificate_example') do - its('stdout.strip') { should_not eq '' } -end - -# Test the min_master_version output -describe command('terraform output min_master_version_example') do - its('stdout.strip') { should eq ENV['KUBERNETES_VERSION'] } -end - -# Test the master_version output -describe command('terraform output master_version_example') do - its('stdout.strip') { should eq ENV['KUBERNETES_VERSION'] } -end - -# Test the network_policy output -describe command('terraform output network_policy_example') do - its('stdout.strip') { should eq 'true' } -end - -# Test the http_load_balancing_enabled output -describe command('terraform output http_load_balancing_example') do - its('stdout.strip') { should eq 'false' } -end - -# Test the horizontal_pod_autoscaling_enabled output -describe command('terraform output horizontal_pod_autoscaling_example') do - its('stdout.strip') { should eq 'true' } -end - -# Test the kubernetes_dashboard_enabled output -describe command('terraform output kubernetes_dashboard_example') do - its('stdout.strip') { should eq 'true' } -end - -# Test the node_pools_names output -describe command('terraform output node_pools_names_example') do - its('stdout.strip') { should eq 'pool-01,' } -end - -# Test the node_pools_versions output -describe command('terraform output node_pools_versions_example') do - its('stdout.strip') { should eq ENV['KUBERNETES_VERSION'] + ',' } -end diff --git a/test/integration/gcloud/test/support/google_cloud.rb b/test/integration/gcloud/test/support/google_cloud.rb deleted file mode 100644 index 1f88781a4b..0000000000 --- a/test/integration/gcloud/test/support/google_cloud.rb +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -require 'googleauth' -require 'google/apis/compute_v1' - -def google_compute_service - Google::Apis::ComputeV1::ComputeService.new.tap do |service| - service.authorization = Google::Auth.get_application_default(['https://www.googleapis.com/auth/cloud-platform']) - end -end diff --git a/test/integration/gcloud/zonal_config.sh b/test/integration/gcloud/zonal_config.sh deleted file mode 100644 index 88c4ea53ba..0000000000 --- a/test/integration/gcloud/zonal_config.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright 2018 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -################################################################# -# PLEASE FILL THE VARIABLES WITH VALID VALUES FOR TESTING # -# DO NOT REMOVE ANY OF THE VARIABLES # -################################################################# - -export ZONE="us-east4-a" -export ADDITIONAL_ZONES='"us-east4-b","us-east4-c"' -export ZONAL_LOCATION="$ZONE" - diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb new file mode 100644 index 0000000000..62f0c108eb --- /dev/null +++ b/test/integration/node_pool/controls/gcloud.rb @@ -0,0 +1,165 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pools" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has 2" do + expect(node_pools.count).to eq 2 + end + + describe "pool-01" do + let(:node_pool) { node_pools.select { |p| p['name'] == "pool-01" }.first } + + it "exists" do + expect(node_pool).not_to be_nil + expect(node_pool['name']).to eq "pool-01" + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 2 + end + + it "has autorepair enabled" do + expect(node_pool['management']['autoRepair']).to eq true + end + + it "has automatic upgrades enabled" do + expect(node_pool['management']['autoUpgrade']).to eq true + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "all-pools-example" => "true", + "pool-01-example" => "true", + "cluster_name" => cluster_name, + "node_pool" => "pool-01", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to match_array([ + "all-node-example", + "pool-01-example", + "gke-node-pool-cluster", + "gke-node-pool-cluster-pool-01", + ]) + end + end + + describe "pool-02" do + let(:node_pool) { node_pools.select { |p| p['name'] == "pool-02" }.first } + + it "exists" do + expect(node_pool).not_to be_nil + expect(node_pool['name']).to eq "pool-02" + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 1 + end + + it "has the expected maximum node count" do + expect(node_pool['autoscaling']['maxNodeCount']).to eq 3 + end + + it "has the expected disk size" do + expect(node_pool['config']['diskSizeGb']).to eq 30 + end + + it "has the expected disk type" do + expect(node_pool['config']['diskType']).to eq "pd-standard" + end + + it "has the expected image type" do + expect(node_pool['config']['imageType']).to eq "COS" + end + + it "has autorepair disabled" do + expect(node_pool['management']['autoRepair']).to eq nil + end + + it "has automatic upgrades disabled" do + expect(node_pool['management']['autoUpgrade']).to eq nil + end + + it "has the right service account" do + expect(node_pool['config']['serviceAccount']).to eq "default" + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "all-pools-example" => "true", + "cluster_name" => cluster_name, + "node_pool" => "pool-02", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to match_array([ + "all-node-example", + "gke-node-pool-cluster", + "gke-node-pool-cluster-pool-02", + ]) + end + end + end + end +end diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb new file mode 100644 index 0000000000..762046b6d1 --- /dev/null +++ b/test/integration/node_pool/controls/kubectl.rb @@ -0,0 +1,87 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'kubeclient' +require 'rest-client' + +require 'base64' + +kubernetes_endpoint = attribute('kubernetes_endpoint') +client_token = attribute('client_token') +ca_certificate = attribute('ca_certificate') + +control "kubectl" do + title "Kubernetes configuration" + + describe "kubernetes" do + let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } + let(:client) do + cert_store = OpenSSL::X509::Store.new + cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) + Kubeclient::Client.new( + kubernetes_http_endpoint, + "v1", + ssl_options: { + cert_store: cert_store, + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + }, + auth_options: { + bearer_token: Base64.decode64(client_token), + }, + ) + end + + describe "nodes" do + let(:all_nodes) { client.get_nodes } + let(:taints) { nodes.first.spec.taints.map { |t| t.to_h.select { |k, v| [:effect, :key, :value].include?(k.to_sym) } } } + + describe "pool-01" do + let(:nodes) do + all_nodes.select { |n| n.metadata.labels.node_pool == "pool-01" } + end + + it "has the expected taints" do + expect(taints).to eq([ + { + effect: "PreferNoSchedule", + key: "all-pools-example", + value: "true", + }, + { + effect: "PreferNoSchedule", + key: "pool-01-example", + value: "true", + }, + ]) + end + end + + describe "pool-02" do + let(:nodes) do + all_nodes.select { |n| n.metadata.labels.node_pool == "pool-02" } + end + + it "has the expected taints" do + expect(taints).to eq([ + { + effect: "PreferNoSchedule", + key: "all-pools-example", + value: "true", + }, + ]) + end + end + end + end +end diff --git a/test/integration/node_pool/inspec.yml b/test/integration/node_pool/inspec.yml new file mode 100644 index 0000000000..3a94f4b479 --- /dev/null +++ b/test/integration/node_pool/inspec.yml @@ -0,0 +1,40 @@ +name: node_pool +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: ca_certificate + required: true + type: string diff --git a/test/integration/simple_regional/controls/gcloud.rb b/test/integration/simple_regional/controls/gcloud.rb new file mode 100644 index 0000000000..1613212de0 --- /dev/null +++ b/test/integration/simple_regional/controls/gcloud.rb @@ -0,0 +1,146 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +region = attribute('region') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') +master_kubernetes_version = attribute('master_kubernetes_version') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['zone']).to eq region + end + + it "has the expected initial cluster version" do + expect(data['initialClusterVersion']).to eq master_kubernetes_version + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => { + "disabled" => true, + }, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pool) { data['nodePools'].reject { |p| p['name'] == "default-pool" }.first } + + it "is running the expected version of Kubernetes" do + expect(node_pool['version']).to eq master_kubernetes_version + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 1 + end + + it "has the expected maximum node count" do + expect(node_pool['autoscaling']['maxNodeCount']).to eq 100 + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has the expected disk size" do + expect(node_pool['config']['diskSizeGb']).to eq 100 + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to eq([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]) + end + + it "has autorepair enabled" do + expect(node_pool['management']['autoRepair']).to eq true + end + + it "has autoupgrade enabled" do + expect(node_pool['management']['autoUpgrade']).to eq true + end + end + end +end diff --git a/test/integration/simple_regional/inspec.yml b/test/integration/simple_regional/inspec.yml new file mode 100644 index 0000000000..01aaee96c9 --- /dev/null +++ b/test/integration/simple_regional/inspec.yml @@ -0,0 +1,37 @@ +name: simple_regional +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/simple_zonal/controls/gcloud.rb b/test/integration/simple_zonal/controls/gcloud.rb new file mode 100644 index 0000000000..1c7d1e8fe1 --- /dev/null +++ b/test/integration/simple_zonal/controls/gcloud.rb @@ -0,0 +1,147 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +region = attribute('region') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') +master_kubernetes_version = attribute('master_kubernetes_version') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['zone']).to eq location + expect(data['zone']).not_to eq region + end + + it "has the expected initial cluster version" do + expect(data['initialClusterVersion']).to eq master_kubernetes_version + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => { + "disabled" => true, + }, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pool) { data['nodePools'].reject { |p| p['name'] == "default-pool" }.first } + + it "is running the expected version of Kubernetes" do + expect(node_pool['version']).to eq master_kubernetes_version + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 1 + end + + it "has the expected maximum node count" do + expect(node_pool['autoscaling']['maxNodeCount']).to eq 100 + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has the expected disk size" do + expect(node_pool['config']['diskSizeGb']).to eq 100 + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to eq([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]) + end + + it "has autorepair enabled" do + expect(node_pool['management']['autoRepair']).to eq true + end + + it "has autoupgrade disabled" do + expect(node_pool['management']['autoUpgrade']).to eq nil + end + end + end +end diff --git a/test/integration/simple_zonal/inspec.yml b/test/integration/simple_zonal/inspec.yml new file mode 100644 index 0000000000..898890335e --- /dev/null +++ b/test/integration/simple_zonal/inspec.yml @@ -0,0 +1,37 @@ +name: simple_zonal +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/stub_domains/controls/gcloud.rb b/test/integration/stub_domains/controls/gcloud.rb new file mode 100644 index 0000000000..d6b277a57a --- /dev/null +++ b/test/integration/stub_domains/controls/gcloud.rb @@ -0,0 +1,72 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => { + "disabled" => true, + }, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + end +end diff --git a/test/integration/stub_domains/controls/kubectl.rb b/test/integration/stub_domains/controls/kubectl.rb new file mode 100644 index 0000000000..1fa048e98d --- /dev/null +++ b/test/integration/stub_domains/controls/kubectl.rb @@ -0,0 +1,88 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'kubeclient' +require 'rest-client' + +require 'base64' + +kubernetes_endpoint = attribute('kubernetes_endpoint') +client_token = attribute('client_token') +ca_certificate = attribute('ca_certificate') + +control "kubectl" do + title "Kubernetes configuration" + + describe "kubernetes" do + let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } + let(:client) do + cert_store = OpenSSL::X509::Store.new + cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) + Kubeclient::Client.new( + kubernetes_http_endpoint, + "v1", + ssl_options: { + cert_store: cert_store, + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + }, + auth_options: { + bearer_token: Base64.decode64(client_token), + }, + ) + end + + describe "configmap" do + describe "kube-dns" do + let(:kubedns_configmap) { client.get_config_map("kube-dns", "kube-system") } + + it "is created by Terraform" do + expect(kubedns_configmap.metadata.labels.maintained_by).to eq "terraform" + end + + it "reflects the stub_domains configuration" do + expect(JSON.parse(kubedns_configmap.data.stubDomains)).to eq({ + "example.com" => [ + "10.254.154.11", + "10.254.154.12", + ], + "example.net" => [ + "10.254.154.11", + "10.254.154.12", + ], + }) + end + end + + describe "ipmasq" do + let(:ipmasq_configmap) { client.get_config_map("ip-masq-agent", "kube-system") } + + it "is created by Terraform" do + expect(ipmasq_configmap.metadata.labels.maintained_by).to eq "terraform" + end + + it "is configured properly" do + expect(YAML.load(ipmasq_configmap.data.config)).to eq({ + "nonMasqueradeCIDRs" => [ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ], + "resyncInterval" => "60s", + "masqLinkLocal" => false, + }) + end + end + end + end +end diff --git a/test/integration/stub_domains/inspec.yml b/test/integration/stub_domains/inspec.yml new file mode 100644 index 0000000000..e04eaf338f --- /dev/null +++ b/test/integration/stub_domains/inspec.yml @@ -0,0 +1,40 @@ +name: stub_domain +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: ca_certificate + required: true + type: string diff --git a/test/make.sh b/test/make.sh index a48cd91bbf..e696ce8fa5 100755 --- a/test/make.sh +++ b/test/make.sh @@ -78,7 +78,7 @@ function check_shell() { # There are some exclusions function check_trailing_whitespace() { echo "The following lines have trailing whitespace" - grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude="*.png" --exclude="*.pyc" --exclude-dir=".git" . + grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude-dir=".kitchen" --exclude="*.png" --exclude="*.pyc" --exclude-dir=".git" . rc=$? if [ $rc = 0 ]; then exit 1 @@ -88,9 +88,22 @@ function check_trailing_whitespace() { function generate_docs() { echo "Generating markdown docs with terraform-docs" TMPFILE=$(mktemp) + #shellcheck disable=2006,2086 for j in `for i in $(find . -type f | grep \.tf$) ; do dirname $i ; done | sort -u` ; do terraform-docs markdown "$j" > "$TMPFILE" python helpers/combine_docfiles.py "$j"/README.md "$TMPFILE" done rm -f "$TMPFILE" } + +function prepare_test_variables() { + echo "Preparing terraform.tfvars files for integration tests" + #shellcheck disable=2044 + for i in $(find ./test/fixtures -type f -name terraform.tfvars.sample); do + destination=${i/%.sample/} + if [ ! -f "${destination}" ]; then + cp "${i}" "${destination}" + echo "${destination} has been created. Please edit it to reflect your GCP configuration." + fi + done +} From 8f7ee983b86b3ed447bbbb9aab367ec9118a3ee7 Mon Sep 17 00:00:00 2001 From: Jason Berlinsky Date: Tue, 23 Oct 2018 13:00:34 -0400 Subject: [PATCH 2/5] Switch to google-beta provider --- auth.tf | 4 +++- cluster_regional.tf | 2 ++ cluster_zonal.tf | 2 ++ examples/deploy_service/main.tf | 6 ++++-- examples/node_pool/main.tf | 6 ++++-- examples/simple_regional/main.tf | 6 ++++-- examples/simple_zonal/main.tf | 6 ++++-- examples/stub_domains/main.tf | 6 ++++-- main.tf | 10 ++++++---- networks.tf | 7 +++++-- 10 files changed, 38 insertions(+), 17 deletions(-) diff --git a/auth.tf b/auth.tf index 4a664d99f5..f2bd23924b 100644 --- a/auth.tf +++ b/auth.tf @@ -17,7 +17,9 @@ /****************************************** Retrieve authentication token *****************************************/ -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} /****************************************** Configure provider diff --git a/cluster_regional.tf b/cluster_regional.tf index 10ad08dc55..2c590df4fe 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -19,6 +19,7 @@ *****************************************/ resource "google_container_cluster" "primary" { count = "${var.regional ? 1 : 0}" + provider = "google-beta" name = "${var.name}" description = "${var.description}" project = "${var.project_id}" @@ -88,6 +89,7 @@ resource "google_container_cluster" "primary" { *****************************************/ resource "google_container_node_pool" "pools" { count = "${var.regional ? length(var.node_pools) : 0}" + provider = "google-beta" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" region = "${var.region}" diff --git a/cluster_zonal.tf b/cluster_zonal.tf index adbc22cbd3..2775457b9f 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -19,6 +19,7 @@ *****************************************/ resource "google_container_cluster" "zonal_primary" { count = "${var.regional ? 0 : 1}" + provider = "google-beta" name = "${var.name}" description = "${var.description}" project = "${var.project_id}" @@ -88,6 +89,7 @@ resource "google_container_cluster" "zonal_primary" { *****************************************/ resource "google_container_node_pool" "zonal_pools" { count = "${var.regional ? 0 : length(var.node_pools)}" + provider = "google-beta" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" zone = "${var.zones[0]}" diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index 2dd7b2489d..1c948e1945 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google" { +provider "google-beta" { credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -26,7 +26,9 @@ provider "kubernetes" { cluster_ca_certificate = "${base64decode(module.gke.ca_certificate)}" } -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} module "gke" { source = "../../" diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 7140a2efa8..840b35b7eb 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google" { +provider "google-beta" { credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -93,4 +93,6 @@ module "gke" { } } -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 40cc858640..dba63d3bbc 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google" { +provider "google-beta" { credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -31,4 +31,6 @@ module "gke" { ip_range_services = "${var.ip_range_services}" } -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal/main.tf index 3c8d918fb4..f12195c8c1 100644 --- a/examples/simple_zonal/main.tf +++ b/examples/simple_zonal/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google" { +provider "google-beta" { credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -32,4 +32,6 @@ module "gke" { ip_range_services = "${var.ip_range_services}" } -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index 55e50b98d5..09e2ac6bec 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -14,7 +14,7 @@ * limitations under the License. */ -provider "google" { +provider "google-beta" { credentials = "${file(var.credentials_path)}" region = "${var.region}" } @@ -43,4 +43,6 @@ module "gke" { } } -data "google_client_config" "default" {} +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/main.tf b/main.tf index c9107d18ec..2a003c87da 100644 --- a/main.tf +++ b/main.tf @@ -18,8 +18,9 @@ Get available zones in region *****************************************/ data "google_compute_zones" "available" { - project = "${var.project_id}" - region = "${var.region}" + provider = "google-beta" + project = "${var.project_id}" + region = "${var.region}" } resource "random_shuffle" "available_zones" { @@ -146,6 +147,7 @@ locals { Get available container engine versions *****************************************/ data "google_container_engine_versions" "region" { - zone = "${data.google_compute_zones.available.names[0]}" - project = "${var.project_id}" + provider = "google-beta" + zone = "${data.google_compute_zones.available.names[0]}" + project = "${var.project_id}" } diff --git a/networks.tf b/networks.tf index 47fee1bec3..f305a63833 100644 --- a/networks.tf +++ b/networks.tf @@ -13,12 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + data "google_compute_network" "gke_network" { - name = "${var.network}" - project = "${local.network_project_id}" + provider = "google-beta" + name = "${var.network}" + project = "${local.network_project_id}" } data "google_compute_subnetwork" "gke_subnetwork" { + provider = "google-beta" name = "${var.subnetwork}" region = "${var.region}" project = "${local.network_project_id}" From 461f07514c82aab821b9fe8994609d9ddcddd31e Mon Sep 17 00:00:00 2001 From: Jason Berlinsky Date: Tue, 23 Oct 2018 14:26:08 -0400 Subject: [PATCH 3/5] Add support for private cluster configuration --- README.md | 4 + cluster_regional.tf | 4 +- cluster_regional_private.tf | 143 ++++++++++++++++++++++++++++++++++++ cluster_zonal.tf | 4 +- cluster_zonal_private.tf | 143 ++++++++++++++++++++++++++++++++++++ main.tf | 139 ++++++++++++++++++++++------------- variables.tf | 20 +++++ 7 files changed, 401 insertions(+), 56 deletions(-) create mode 100644 cluster_regional_private.tf create mode 100644 cluster_zonal_private.tf diff --git a/README.md b/README.md index 72a1088955..9520847fae 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,10 @@ Then perform the following commands on the root folder: | node_pools_taints | Map of lists containing node taints by node-pool name | map | `` | no | | node_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `` | no | | non_masquerade_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list | `` | no | +| private | (Beta) Provision as a private cluster | string | `false` | no | +| private_cluster_config_enable_private_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | string | `false` | no | +| private_cluster_config_enable_private_nodes | (Beta) Whether nodes have internal IP addresses only | string | `false` | no | +| private_cluster_config_master_ipv4_cidr_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `10.0.0.0/28` | no | | project_id | The project ID to host the cluster in (required) | string | - | yes | | region | The region to host the cluster in (required) | string | - | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | string | `true` | no | diff --git a/cluster_regional.tf b/cluster_regional.tf index 2c590df4fe..27e317d41b 100644 --- a/cluster_regional.tf +++ b/cluster_regional.tf @@ -18,8 +18,8 @@ Create regional cluster *****************************************/ resource "google_container_cluster" "primary" { - count = "${var.regional ? 1 : 0}" provider = "google-beta" + count = "${(local.cluster_deployment_type == "regional") ? 1 : 0 }" name = "${var.name}" description = "${var.description}" project = "${var.project_id}" @@ -88,8 +88,8 @@ resource "google_container_cluster" "primary" { Create regional node pools *****************************************/ resource "google_container_node_pool" "pools" { - count = "${var.regional ? length(var.node_pools) : 0}" provider = "google-beta" + count = "${(local.cluster_deployment_type == "regional") ? length(var.node_pools) : 0 }" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" region = "${var.region}" diff --git a/cluster_regional_private.tf b/cluster_regional_private.tf new file mode 100644 index 0000000000..8b5e123c65 --- /dev/null +++ b/cluster_regional_private.tf @@ -0,0 +1,143 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + Create regional cluster + *****************************************/ +resource "google_container_cluster" "primary_private" { + provider = "google-beta" + count = "${(local.cluster_deployment_type == "regional_private") ? 1 : 0 }" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + region = "${var.region}" + additional_zones = ["${coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result))}"] + + network = "${data.google_compute_network.gke_network.self_link}" + subnetwork = "${data.google_compute_subnetwork.gke_subnetwork.self_link}" + min_master_version = "${local.kubernetes_version}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = "${var.master_authorized_networks_config}" + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", "")}" + } + } + + private_cluster_config { + enable_private_endpoint = "${var.private_cluster_config_enable_private_endpoint}" + enable_private_nodes = "${var.private_cluster_config_enable_private_nodes}" + master_ipv4_cidr_block = "${var.private_cluster_config_master_ipv4_cidr_block}" + } +} + +/****************************************** + Create regional node pools + *****************************************/ +resource "google_container_node_pool" "pools_private" { + provider = "google-beta" + count = "${(local.cluster_deployment_type == "regional_private") ? length(var.node_pools) : 0 }" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + region = "${var.region}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + initial_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", true)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = "${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}" + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", "")}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.primary"] +} diff --git a/cluster_zonal.tf b/cluster_zonal.tf index 2775457b9f..dc3a3aad00 100644 --- a/cluster_zonal.tf +++ b/cluster_zonal.tf @@ -18,8 +18,8 @@ Create zonal cluster *****************************************/ resource "google_container_cluster" "zonal_primary" { - count = "${var.regional ? 0 : 1}" provider = "google-beta" + count = "${(local.cluster_deployment_type == "zonal") ? 1 : 0 }" name = "${var.name}" description = "${var.description}" project = "${var.project_id}" @@ -88,8 +88,8 @@ resource "google_container_cluster" "zonal_primary" { Create zonal node pools *****************************************/ resource "google_container_node_pool" "zonal_pools" { - count = "${var.regional ? 0 : length(var.node_pools)}" provider = "google-beta" + count = "${(local.cluster_deployment_type == "zonal") ? length(var.node_pools) : 0 }" name = "${lookup(var.node_pools[count.index], "name")}" project = "${var.project_id}" zone = "${var.zones[0]}" diff --git a/cluster_zonal_private.tf b/cluster_zonal_private.tf new file mode 100644 index 0000000000..3f93cf1fae --- /dev/null +++ b/cluster_zonal_private.tf @@ -0,0 +1,143 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/****************************************** + Create zonal cluster + *****************************************/ +resource "google_container_cluster" "zonal_primary_private" { + provider = "google-beta" + count = "${(local.cluster_deployment_type == "zonal_private") ? 1 : 0 }" + name = "${var.name}" + description = "${var.description}" + project = "${var.project_id}" + + zone = "${var.zones[0]}" + additional_zones = "${slice(var.zones,1,length(var.zones))}" + + network = "${data.google_compute_network.gke_network.self_link}" + subnetwork = "${data.google_compute_subnetwork.gke_subnetwork.self_link}" + min_master_version = "${local.kubernetes_version}" + + logging_service = "${var.logging_service}" + monitoring_service = "${var.monitoring_service}" + + master_authorized_networks_config = "${var.master_authorized_networks_config}" + + addons_config { + http_load_balancing { + disabled = "${var.http_load_balancing ? 0 : 1}" + } + + horizontal_pod_autoscaling { + disabled = "${var.horizontal_pod_autoscaling ? 0 : 1}" + } + + kubernetes_dashboard { + disabled = "${var.kubernetes_dashboard ? 0 : 1}" + } + + network_policy_config { + disabled = "${var.network_policy ? 0 : 1}" + } + } + + ip_allocation_policy { + cluster_secondary_range_name = "${var.ip_range_pods}" + services_secondary_range_name = "${var.ip_range_services}" + } + + maintenance_policy { + daily_maintenance_window { + start_time = "${var.maintenance_start_time}" + } + } + + lifecycle { + ignore_changes = ["node_pool"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + node_pool { + name = "default-pool" + + node_config { + service_account = "${lookup(var.node_pools[0], "service_account", "")}" + } + } + + private_cluster_config { + enable_private_endpoint = "${var.private_cluster_config_enable_private_endpoint}" + enable_private_nodes = "${var.private_cluster_config_enable_private_nodes}" + master_ipv4_cidr_block = "${var.private_cluster_config_master_ipv4_cidr_block}" + } +} + +/****************************************** + Create zonal node pools + *****************************************/ +resource "google_container_node_pool" "zonal_pools_private" { + provider = "google-beta" + count = "${(local.cluster_deployment_type == "zonal_private") ? length(var.node_pools) : 0 }" + name = "${lookup(var.node_pools[count.index], "name")}" + project = "${var.project_id}" + zone = "${var.zones[0]}" + cluster = "${var.name}" + version = "${lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup(var.node_pools[count.index], "version", local.node_version)}" + initial_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + + autoscaling { + min_node_count = "${lookup(var.node_pools[count.index], "min_count", 1)}" + max_node_count = "${lookup(var.node_pools[count.index], "max_count", 100)}" + } + + management { + auto_repair = "${lookup(var.node_pools[count.index], "auto_repair", true)}" + auto_upgrade = "${lookup(var.node_pools[count.index], "auto_upgrade", false)}" + } + + node_config { + image_type = "${lookup(var.node_pools[count.index], "image_type", "COS")}" + machine_type = "${lookup(var.node_pools[count.index], "machine_type", "n1-standard-2")}" + labels = "${merge(map("cluster_name", var.name), map("node_pool", lookup(var.node_pools[count.index], "name")), var.node_pools_labels["all"], var.node_pools_labels[lookup(var.node_pools[count.index], "name")])}" + taint = "${concat(var.node_pools_taints["all"], var.node_pools_taints[lookup(var.node_pools[count.index], "name")])}" + tags = "${concat(list("gke-${var.name}"), list("gke-${var.name}-${lookup(var.node_pools[count.index], "name")}"), var.node_pools_tags["all"], var.node_pools_tags[lookup(var.node_pools[count.index], "name")])}" + + disk_size_gb = "${lookup(var.node_pools[count.index], "disk_size_gb", 100)}" + disk_type = "${lookup(var.node_pools[count.index], "disk_type", "pd-standard")}" + service_account = "${lookup(var.node_pools[count.index], "service_account", "")}" + + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + lifecycle { + ignore_changes = ["initial_node_count"] + } + + timeouts { + create = "30m" + update = "30m" + delete = "30m" + } + + depends_on = ["google_container_cluster.zonal_primary"] +} diff --git a/main.tf b/main.tf index 2a003c87da..f7d6e54a23 100644 --- a/main.tf +++ b/main.tf @@ -34,113 +34,148 @@ locals { custom_kube_dns_config = "${length(keys(var.stub_domains)) > 0 ? true : false}" network_project_id = "${var.network_project_id != "" ? var.network_project_id : var.project_id}" - cluster_type = "${var.regional ? "regional" : "zonal"}" + cluster_type = "${var.regional ? "regional" : "zonal"}" + cluster_deployment_type = "${var.private ? join("_", list(local.cluster_type, "private")) : local.cluster_type}" cluster_type_output_name = { - regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.name, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.name, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.name, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.name, list("")), 0)}" } cluster_type_output_location = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.region, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.zone, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.zone, list("")), 0)}" } cluster_type_output_region = { - regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" - zonal = "${var.region}" + regional = "${element(concat(google_container_cluster.primary.*.region, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.region, list("")), 0)}" + zonal = "${var.region}" + zonal_private = "${var.region}" } - cluster_type_output_regional_zones = "${concat(google_container_cluster.primary.*.additional_zones, list(list()))}" - cluster_type_output_zonal_zones = "${concat(google_container_cluster.zonal_primary.*.additional_zones, list(list()))}" + cluster_type_output_regional_zones = "${concat(google_container_cluster.primary.*.additional_zones, list(list()))}" + cluster_type_output_regional_private_zones = "${concat(google_container_cluster.primary_private.*.additional_zones, list(list()))}" + cluster_type_output_zonal_zones = "${concat(google_container_cluster.zonal_primary.*.additional_zones, list(list()))}" + cluster_type_output_zonal_private_zones = "${concat(google_container_cluster.zonal_primary_private.*.additional_zones, list(list()))}" cluster_type_output_zones = { - regional = "${local.cluster_type_output_regional_zones[0]}" - zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones[0])}" + regional = "${local.cluster_type_output_regional_zones[0]}" + regional_private = "${local.cluster_type_output_regional_private_zones[0]}" + zonal = "${concat(google_container_cluster.zonal_primary.*.zone, local.cluster_type_output_zonal_zones[0])}" + zonal_private = "${concat(google_container_cluster.zonal_primary_private.*.zone, local.cluster_type_output_zonal_private_zones[0])}" } cluster_type_output_endpoint = { - regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.endpoint, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.endpoint, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.endpoint, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.endpoint, list("")), 0)}" } cluster_type_output_master_auth = { - regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" - zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + regional = "${concat(google_container_cluster.primary.*.master_auth, list())}" + regional_private = "${concat(google_container_cluster.primary_private.*.master_auth, list())}" + zonal = "${concat(google_container_cluster.zonal_primary.*.master_auth, list())}" + zonal_private = "${concat(google_container_cluster.zonal_primary_private.*.master_auth, list())}" } cluster_type_output_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.master_version, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.master_version, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.master_version, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.master_version, list("")), 0)}" } cluster_type_output_min_master_version = { - regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.min_master_version, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.min_master_version, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.min_master_version, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.min_master_version, list("")), 0)}" } cluster_type_output_logging_service = { - regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.logging_service, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.logging_service, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.logging_service, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.logging_service, list("")), 0)}" } cluster_type_output_monitoring_service = { - regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.monitoring_service, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.monitoring_service, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.monitoring_service, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.monitoring_service, list("")), 0)}" } cluster_type_output_network_policy_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.addons_config.0.network_policy_config.0.disabled, list("")), 0)}" } cluster_type_output_http_load_balancing_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.addons_config.0.http_load_balancing.0.disabled, list("")), 0)}" } cluster_type_output_horizontal_pod_autoscaling_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.addons_config.0.horizontal_pod_autoscaling.0.disabled, list("")), 0)}" } cluster_type_output_kubernetes_dashboard_enabled = { - regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" - zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional = "${element(concat(google_container_cluster.primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + regional_private = "${element(concat(google_container_cluster.primary_private.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + zonal = "${element(concat(google_container_cluster.zonal_primary.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" + zonal_private = "${element(concat(google_container_cluster.zonal_primary_private.*.addons_config.0.kubernetes_dashboard.0.disabled, list("")), 0)}" } cluster_type_output_node_pools_names = { - regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + regional = "${concat(google_container_node_pool.pools.*.name, list(""))}" + regional_private = "${concat(google_container_node_pool.pools_private.*.name, list(""))}" + zonal = "${concat(google_container_node_pool.zonal_pools.*.name, list(""))}" + zonal_private = "${concat(google_container_node_pool.zonal_pools_private.*.name, list(""))}" } cluster_type_output_node_pools_versions = { - regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" - zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + regional = "${concat(google_container_node_pool.pools.*.version, list(""))}" + regional_private = "${concat(google_container_node_pool.pools_private.*.version, list(""))}" + zonal = "${concat(google_container_node_pool.zonal_pools.*.version, list(""))}" + zonal_private = "${concat(google_container_node_pool.zonal_pools_private.*.version, list(""))}" } - cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_type]}" + cluster_master_auth_list_layer1 = "${local.cluster_type_output_master_auth[local.cluster_deployment_type]}" cluster_master_auth_list_layer2 = "${local.cluster_master_auth_list_layer1[0]}" cluster_master_auth_map = "${local.cluster_master_auth_list_layer2[0]}" # cluster locals - cluster_name = "${local.cluster_type_output_name[local.cluster_type]}" - cluster_location = "${local.cluster_type_output_location[local.cluster_type]}" - cluster_region = "${local.cluster_type_output_region[local.cluster_type]}" - cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_type])}" - cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_type]}" + cluster_name = "${local.cluster_type_output_name[local.cluster_deployment_type]}" + cluster_location = "${local.cluster_type_output_location[local.cluster_deployment_type]}" + cluster_region = "${local.cluster_type_output_region[local.cluster_deployment_type]}" + cluster_zones = "${sort(local.cluster_type_output_zones[local.cluster_deployment_type])}" + cluster_endpoint = "${local.cluster_type_output_endpoint[local.cluster_deployment_type]}" cluster_ca_certificate = "${lookup(local.cluster_master_auth_map, "cluster_ca_certificate")}" - cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_type]}" - cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_type]}" - cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_type]}" - cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_type]}" - cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_type]}" - cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_type]}" - - cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_type] ? false : true}" - cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_type] ? false : true}" - cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_type] ? false : true}" - cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_type] ? false : true}" + cluster_master_version = "${local.cluster_type_output_master_version[local.cluster_deployment_type]}" + cluster_min_master_version = "${local.cluster_type_output_min_master_version[local.cluster_deployment_type]}" + cluster_logging_service = "${local.cluster_type_output_logging_service[local.cluster_deployment_type]}" + cluster_monitoring_service = "${local.cluster_type_output_monitoring_service[local.cluster_deployment_type]}" + cluster_node_pools_names = "${local.cluster_type_output_node_pools_names[local.cluster_deployment_type]}" + cluster_node_pools_versions = "${local.cluster_type_output_node_pools_versions[local.cluster_deployment_type]}" + + cluster_network_policy_enabled = "${local.cluster_type_output_network_policy_enabled[local.cluster_deployment_type] ? false : true}" + cluster_http_load_balancing_enabled = "${local.cluster_type_output_http_load_balancing_enabled[local.cluster_deployment_type] ? false : true}" + cluster_horizontal_pod_autoscaling_enabled = "${local.cluster_type_output_horizontal_pod_autoscaling_enabled[local.cluster_deployment_type] ? false : true}" + cluster_kubernetes_dashboard_enabled = "${local.cluster_type_output_kubernetes_dashboard_enabled[local.cluster_deployment_type] ? false : true}" } /****************************************** diff --git a/variables.tf b/variables.tf index b274771bb1..482813e68f 100644 --- a/variables.tf +++ b/variables.tf @@ -189,3 +189,23 @@ variable "monitoring_service" { description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" default = "monitoring.googleapis.com" } + +variable "private" { + description = "(Beta) Provision as a private cluster" + default = false +} + +variable "private_cluster_config_enable_private_endpoint" { + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + default = false +} + +variable "private_cluster_config_enable_private_nodes" { + description = "(Beta) Whether nodes have internal IP addresses only" + default = false +} + +variable "private_cluster_config_master_ipv4_cidr_block" { + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} From 13947fd5c81f397229688e4b2ee4d7967f202ae3 Mon Sep 17 00:00:00 2001 From: Jason Berlinsky Date: Tue, 23 Oct 2018 14:26:18 -0400 Subject: [PATCH 4/5] Add private cluster configuration tests and examples --- .kitchen.yml | 28 ++++ examples/simple_regional_private/README.md | 42 +++++ examples/simple_regional_private/main.tf | 46 ++++++ examples/simple_regional_private/outputs.tf | 69 ++++++++ examples/simple_regional_private/variables.tf | 43 +++++ examples/simple_zonal_private/README.md | 43 +++++ examples/simple_zonal_private/main.tf | 47 ++++++ examples/simple_zonal_private/outputs.tf | 69 ++++++++ examples/simple_zonal_private/variables.tf | 48 ++++++ .../terraform.tfvars.sample | 7 + .../terraform.tfvars.sample | 8 + .../controls/gcloud.rb | 151 +++++++++++++++++ .../simple_regional_private/inspec.yml | 37 +++++ .../simple_zonal_private/controls/gcloud.rb | 152 ++++++++++++++++++ .../simple_zonal_private/inspec.yml | 37 +++++ 15 files changed, 827 insertions(+) create mode 100644 examples/simple_regional_private/README.md create mode 100644 examples/simple_regional_private/main.tf create mode 100644 examples/simple_regional_private/outputs.tf create mode 100644 examples/simple_regional_private/variables.tf create mode 100644 examples/simple_zonal_private/README.md create mode 100644 examples/simple_zonal_private/main.tf create mode 100644 examples/simple_zonal_private/outputs.tf create mode 100644 examples/simple_zonal_private/variables.tf create mode 100644 test/fixtures/simple_regional_private/terraform.tfvars.sample create mode 100644 test/fixtures/simple_zonal_private/terraform.tfvars.sample create mode 100644 test/integration/simple_regional_private/controls/gcloud.rb create mode 100644 test/integration/simple_regional_private/inspec.yml create mode 100644 test/integration/simple_zonal_private/controls/gcloud.rb create mode 100644 test/integration/simple_zonal_private/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 4dbcbf0265..58e1c58085 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -80,6 +80,34 @@ suites: backend: local provisioner: name: terraform + - name: "simple_regional_private" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/simple_regional_private + variable_files: + - test/fixtures/simple_regional_private/terraform.tfvars + verifier: + name: terraform + systems: + - name: simple_regional_private + backend: local + provisioner: + name: terraform + - name: "simple_zonal_private" + driver: + name: "terraform" + command_timeout: 1800 + root_module_directory: examples/simple_zonal_private + variable_files: + - test/fixtures/simple_zonal_private/terraform.tfvars + verifier: + name: terraform + systems: + - name: simple_zonal_private + backend: local + provisioner: + name: terraform - name: "stub_domains" driver: name: "terraform" diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md new file mode 100644 index 0000000000..c0d60db5d5 --- /dev/null +++ b/examples/simple_regional_private/README.md @@ -0,0 +1,42 @@ +# Simple Regional Cluster + +This example illustrates how to create a simple cluster. + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | Cluster endpoint | +| location | Cluster location | +| master_kubernetes_version | The master Kubernetes version | +| network | Network the cluster is provisioned in | +| project_id | | +| region | | +| subnetwork | Subnetwork the cluster is provisioned in | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf new file mode 100644 index 0000000000..2921bad387 --- /dev/null +++ b/examples/simple_regional_private/main.tf @@ -0,0 +1,46 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google-beta" { + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +module "gke" { + source = "../../" + project_id = "${var.project_id}" + name = "simple-regional-cluster-private" + regional = true + region = "${var.region}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + private = true + private_cluster_config_enable_private_endpoint = true + private_cluster_config_enable_private_nodes = true + private_cluster_config_master_ipv4_cidr_block = "172.16.0.0/28" + master_authorized_networks_config = [{ + cidr_blocks = [{ + cidr_block = "10.0.0.0/8" + display_name = "VPC" + }] + }] +} + +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf new file mode 100644 index 0000000000..dd74cf3ca5 --- /dev/null +++ b/examples/simple_regional_private/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "network" { + description = "Network the cluster is provisioned in" + value = "${var.network}" +} + +output "subnetwork" { + description = "Subnetwork the cluster is provisioned in" + value = "${var.subnetwork}" +} + +output "kubernetes_endpoint" { + sensitive = true + description = "Cluster endpoint" + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "location" { + description = "Cluster location" + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/simple_regional_private/variables.tf b/examples/simple_regional_private/variables.tf new file mode 100644 index 0000000000..dc111ef220 --- /dev/null +++ b/examples/simple_regional_private/variables.tf @@ -0,0 +1,43 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md new file mode 100644 index 0000000000..0b66ab1480 --- /dev/null +++ b/examples/simple_zonal_private/README.md @@ -0,0 +1,43 @@ +# Simple Zonal Cluster + +This example illustrates how to create a simple cluster. + +[^]: (autogen_docs_start) + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| credentials_path | The path to a Google Cloud Service Account credentials file | string | - | yes | +| ip_range_pods | The secondary ip range to use for pods | string | - | yes | +| ip_range_services | The secondary ip range to use for pods | string | - | yes | +| network | The VPC network to host the cluster in | string | - | yes | +| project_id | The project ID to host the cluster in | string | - | yes | +| region | The region to host the cluster in | string | - | yes | +| subnetwork | The subnetwork to host the cluster in | string | - | yes | +| zones | The zone to host the cluster in (required if is a zonal cluster) | list | - | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| client_token | | +| cluster_name | Cluster name | +| ip_range_pods | The secondary IP range used for pods | +| ip_range_services | The secondary IP range used for services | +| kubernetes_endpoint | Cluster endpoint | +| location | Cluster location | +| master_kubernetes_version | The master Kubernetes version | +| network | Network the cluster is provisioned in | +| project_id | | +| region | | +| subnetwork | Subnetwork the cluster is provisioned in | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure \ No newline at end of file diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf new file mode 100644 index 0000000000..01ff50052f --- /dev/null +++ b/examples/simple_zonal_private/main.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +provider "google-beta" { + credentials = "${file(var.credentials_path)}" + region = "${var.region}" +} + +module "gke" { + source = "../../" + project_id = "${var.project_id}" + name = "simple-zonal-cluster-private" + regional = false + region = "${var.region}" + zones = "${var.zones}" + network = "${var.network}" + subnetwork = "${var.subnetwork}" + ip_range_pods = "${var.ip_range_pods}" + ip_range_services = "${var.ip_range_services}" + private = true + private_cluster_config_enable_private_endpoint = true + private_cluster_config_enable_private_nodes = true + private_cluster_config_master_ipv4_cidr_block = "172.16.0.16/28" + master_authorized_networks_config = [{ + cidr_blocks = [{ + cidr_block = "10.0.0.0/8" + display_name = "VPC" + }] + }] +} + +data "google_client_config" "default" { + provider = "google-beta" +} diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf new file mode 100644 index 0000000000..dd74cf3ca5 --- /dev/null +++ b/examples/simple_zonal_private/outputs.tf @@ -0,0 +1,69 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = "${var.project_id}" +} + +output "region" { + value = "${var.region}" +} + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "network" { + description = "Network the cluster is provisioned in" + value = "${var.network}" +} + +output "subnetwork" { + description = "Subnetwork the cluster is provisioned in" + value = "${var.subnetwork}" +} + +output "kubernetes_endpoint" { + sensitive = true + description = "Cluster endpoint" + value = "${module.gke.endpoint}" +} + +output "client_token" { + sensitive = true + value = "${base64encode(data.google_client_config.default.access_token)}" +} + +output "location" { + description = "Cluster location" + value = "${module.gke.location}" +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = "${var.ip_range_pods}" +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = "${var.ip_range_services}" +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = "${module.gke.master_version}" +} diff --git a/examples/simple_zonal_private/variables.tf b/examples/simple_zonal_private/variables.tf new file mode 100644 index 0000000000..fb4755571d --- /dev/null +++ b/examples/simple_zonal_private/variables.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "credentials_path" { + description = "The path to a Google Cloud Service Account credentials file" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "zones" { + type = "list" + description = "The zone to host the cluster in (required if is a zonal cluster)" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} diff --git a/test/fixtures/simple_regional_private/terraform.tfvars.sample b/test/fixtures/simple_regional_private/terraform.tfvars.sample new file mode 100644 index 0000000000..e17012a071 --- /dev/null +++ b/test/fixtures/simple_regional_private/terraform.tfvars.sample @@ -0,0 +1,7 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" diff --git a/test/fixtures/simple_zonal_private/terraform.tfvars.sample b/test/fixtures/simple_zonal_private/terraform.tfvars.sample new file mode 100644 index 0000000000..d469c8925f --- /dev/null +++ b/test/fixtures/simple_zonal_private/terraform.tfvars.sample @@ -0,0 +1,8 @@ +project_id="" +credentials_path="../../credentials.json" +region="us-east4" +zones=["us-east4-b","us-east4-c","us-east4-a"] +network="vpc-01" +subnetwork="us-east4-01" +ip_range_pods="us-east4-01-gke-01-pod" +ip_range_services="us-east4-01-gke-01-service" diff --git a/test/integration/simple_regional_private/controls/gcloud.rb b/test/integration/simple_regional_private/controls/gcloud.rb new file mode 100644 index 0000000000..fa34164a9e --- /dev/null +++ b/test/integration/simple_regional_private/controls/gcloud.rb @@ -0,0 +1,151 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +region = attribute('region') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') +master_kubernetes_version = attribute('master_kubernetes_version') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['zone']).to eq region + end + + it "is private" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected initial cluster version" do + expect(data['initialClusterVersion']).to eq master_kubernetes_version + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => { + "disabled" => true, + }, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pool) { data['nodePools'].reject { |p| p['name'] == "default-pool" }.first } + + it "is running the expected version of Kubernetes" do + expect(node_pool['version']).to eq master_kubernetes_version + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 1 + end + + it "has the expected maximum node count" do + expect(node_pool['autoscaling']['maxNodeCount']).to eq 100 + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has the expected disk size" do + expect(node_pool['config']['diskSizeGb']).to eq 100 + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to eq([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]) + end + + it "has autorepair enabled" do + expect(node_pool['management']['autoRepair']).to eq true + end + + it "has autoupgrade enabled" do + expect(node_pool['management']['autoUpgrade']).to eq true + end + end + end +end diff --git a/test/integration/simple_regional_private/inspec.yml b/test/integration/simple_regional_private/inspec.yml new file mode 100644 index 0000000000..861aab7f57 --- /dev/null +++ b/test/integration/simple_regional_private/inspec.yml @@ -0,0 +1,37 @@ +name: simple_regional_private +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string diff --git a/test/integration/simple_zonal_private/controls/gcloud.rb b/test/integration/simple_zonal_private/controls/gcloud.rb new file mode 100644 index 0000000000..83d4b4773d --- /dev/null +++ b/test/integration/simple_zonal_private/controls/gcloud.rb @@ -0,0 +1,152 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +region = attribute('region') +location = attribute('location') +cluster_name = attribute('cluster_name') +network = attribute('network') +subnetwork = attribute('subnetwork') +ip_range_pods = attribute('ip_range_pods') +ip_range_services = attribute('ip_range_services') +master_kubernetes_version = attribute('master_kubernetes_version') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['zone']).to eq location + expect(data['zone']).not_to eq region + end + + it "is private" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected initial cluster version" do + expect(data['initialClusterVersion']).to eq master_kubernetes_version + end + + it "is in the expected network" do + expect(data['network']).to eq network + end + + it "is in the expected subnetwork" do + expect(data['subnetwork']).to eq subnetwork + end + + it "has the expected secondary ip range for pods" do + expect(data['ipAllocationPolicy']['clusterSecondaryRangeName']).to eq ip_range_pods + end + + it "has the expected secondary ip range for services" do + expect(data['ipAllocationPolicy']['servicesSecondaryRangeName']).to eq ip_range_services + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => { + "disabled" => true, + }, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pool) { data['nodePools'].reject { |p| p['name'] == "default-pool" }.first } + + it "is running the expected version of Kubernetes" do + expect(node_pool['version']).to eq master_kubernetes_version + end + + it "has autoscaling enabled" do + expect(node_pool['autoscaling']['enabled']).to eq true + end + + it "has the expected minimum node count" do + expect(node_pool['autoscaling']['minNodeCount']).to eq 1 + end + + it "has the expected maximum node count" do + expect(node_pool['autoscaling']['maxNodeCount']).to eq 100 + end + + it "is the expected machine type" do + expect(node_pool['config']['machineType']).to eq 'n1-standard-2' + end + + it "has the expected disk size" do + expect(node_pool['config']['diskSizeGb']).to eq 100 + end + + it "has the expected labels" do + expect(node_pool['config']['labels']).to eq({ + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + }) + end + + it "has the expected network tags" do + expect(node_pool['config']['tags']).to eq([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]) + end + + it "has autorepair enabled" do + expect(node_pool['management']['autoRepair']).to eq true + end + + it "has autoupgrade disabled" do + expect(node_pool['management']['autoUpgrade']).to eq nil + end + end + end +end diff --git a/test/integration/simple_zonal_private/inspec.yml b/test/integration/simple_zonal_private/inspec.yml new file mode 100644 index 0000000000..cbd5707cd2 --- /dev/null +++ b/test/integration/simple_zonal_private/inspec.yml @@ -0,0 +1,37 @@ +name: simple_zonal_private +attributes: + - name: project_id + required: true + type: string + - name: region + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: network + required: false + type: string + default: "default" + - name: subnetwork + required: false + type: string + default: "default" + - name: ip_range_pods + required: true + type: string + - name: ip_range_services + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From dfd5792d0cd52d2fbd30e3e6460d7ed867f78340 Mon Sep 17 00:00:00 2001 From: Jason Berlinsky Date: Wed, 24 Oct 2018 11:17:34 -0400 Subject: [PATCH 5/5] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1e411302e..4582ad936d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Changed +* Added suport for private clusters. #21 +* Migrated to [google-beta provider](https://github.com/terraform-providers/terraform-provider-google-beta) to support private clusters. #21 + ## [v0.3.0] - 2018-10-10 ### Changed * Updated network/subnetwork lookup to use data source. #16