From e3a05d800ab985f0b1b47407e1c291797ce1fc1b Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Fri, 4 Oct 2019 18:34:01 +0300 Subject: [PATCH 01/35] Add submodule to set up ACM Added standalone ACM submodule, embedded acm to examples/simple_zonal. Fixes #268 --- examples/simple_zonal/README.md | 4 ++ examples/simple_zonal/acm.tf | 25 +++++++ examples/simple_zonal/outputs.tf | 4 ++ examples/simple_zonal/variables.tf | 17 +++++ modules/acm/.gitignore | 3 + modules/acm/README.md | 41 +++++++++++ modules/acm/main.tf | 90 ++++++++++++++++++++++++ modules/acm/outputs.tf | 4 ++ modules/acm/scripts/kubectl_wrapper.sh | 53 ++++++++++++++ modules/acm/templates/acm-config.yml.tpl | 12 ++++ modules/acm/variables.tf | 30 ++++++++ test/setup/iam.tf | 1 + test/task_helper_functions.sh | 2 +- 13 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 examples/simple_zonal/acm.tf create mode 100644 modules/acm/.gitignore create mode 100644 modules/acm/README.md create mode 100644 modules/acm/main.tf create mode 100644 modules/acm/outputs.tf create mode 100755 modules/acm/scripts/kubectl_wrapper.sh create mode 100644 modules/acm/templates/acm-config.yml.tpl create mode 100644 modules/acm/variables.tf diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md index 691f95c719..265977c33b 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal/README.md @@ -7,6 +7,9 @@ This example illustrates how to create a simple cluster. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| acm\_policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | `"foo-corp"` | no | +| acm\_sync\_branch | Anthos config management Git branch | string | `"1.0.0"` | no | +| acm\_sync\_repo | Anthos config management Git repo | string | `"git@github.com:GoogleCloudPlatform/csp-config-management.git"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | @@ -20,6 +23,7 @@ This example illustrates how to create a simple cluster. | Name | Description | |------|-------------| +| acm\_git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. | | ca\_certificate | | | client\_token | | | cluster\_name | Cluster name | diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal/acm.tf new file mode 100644 index 0000000000..1c38d22c99 --- /dev/null +++ b/examples/simple_zonal/acm.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "acm" { + source = "../../modules/acm" + project_id = var.project_id + location = module.gke.location + cluster_name = module.gke.name + sync_repo = var.acm_sync_repo + sync_branch = var.acm_sync_branch + policy_dir = var.acm_policy_dir +} diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf index 0d972dcd88..143710f8b2 100644 --- a/examples/simple_zonal/outputs.tf +++ b/examples/simple_zonal/outputs.tf @@ -33,3 +33,7 @@ output "service_account" { value = module.gke.service_account } +output "acm_git_creds_public" { + description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository." + value = module.acm.git_creds_public +} \ No newline at end of file diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal/variables.tf index 516116557c..adc92dba17 100644 --- a/examples/simple_zonal/variables.tf +++ b/examples/simple_zonal/variables.tf @@ -48,3 +48,20 @@ variable "ip_range_services" { description = "The secondary ip range to use for pods" } +variable "acm_sync_repo" { + description = "Anthos config management Git repo" + type = string + default = "git@github.com:GoogleCloudPlatform/csp-config-management.git" +} + +variable "acm_sync_branch" { + description = "Anthos config management Git branch" + type = string + default = "1.0.0" +} + +variable "acm_policy_dir" { + description = "Subfolder containing configs in Ahtons config management Git repo" + type = string + default = "foo-corp" +} \ No newline at end of file diff --git a/modules/acm/.gitignore b/modules/acm/.gitignore new file mode 100644 index 0000000000..aee8cb6749 --- /dev/null +++ b/modules/acm/.gitignore @@ -0,0 +1,3 @@ +# This fill will be always downloaded by terraform local-exec command from gc bucket +config-management-operator.yaml +/terraform.tfvars diff --git a/modules/acm/README.md b/modules/acm/README.md new file mode 100644 index 0000000000..99a40bc840 --- /dev/null +++ b/modules/acm/README.md @@ -0,0 +1,41 @@ +# Terraform Kubernetes Engine ACM Submodule + +This module installs Anthos Config Management (ACM) in a Kubernetes cluster. +To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/). + + +## Configure a Service Account + +In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md) +plus **roles/container.admin** role. + +## Usage example + +See [examples/simple_zonal](../../examples/simple_zonal) cluster example. + +## Installation + +Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page. +Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public** +output variable to your account at your Git server. + + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cluster\_name | The name of the cluster. | string | n/a | yes | +| location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes | +| policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes | +| project\_id | The project in which the resource belongs. | string | n/a | yes | +| sync\_branch | Anthos config management Git branch | string | `"master"` | no | +| sync\_repo | Anthos config management Git repo | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. | + + \ No newline at end of file diff --git a/modules/acm/main.tf b/modules/acm/main.tf new file mode 100644 index 0000000000..7a2a6019d4 --- /dev/null +++ b/modules/acm/main.tf @@ -0,0 +1,90 @@ +locals { + cluster_endpoint = "https://${data.google_container_cluster.primary.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate +} + +data "google_container_cluster" "primary" { + name = var.cluster_name + project = var.project_id + location = var.location +} + +data "google_client_config" "default" { +} + +resource "tls_private_key" "git_creds" { + algorithm = "RSA" + rsa_bits = 4096 +} + +resource "null_resource" "acm_operator_config" { + provisioner "local-exec" { + command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" + } + + provisioner "local-exec" { + when = "destroy" + command = "rm -f ${path.module}/config-management-operator.yaml" + } +} + +resource "null_resource" "acm_operator" { + provisioner "local-exec" { + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${path.module}/config-management-operator.yaml" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml" + } + + depends_on = [ + null_resource.acm_operator_config, + data.google_client_config.default, + data.google_container_cluster.primary, + ] +} + +resource "null_resource" "git_creds_secret" { + provisioner "local-exec" { + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${tls_private_key.git_creds.private_key_pem}'" + } + + provisioner "local-exec" { + when = "destroy" + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete secret git-creds -n=config-management-system" + } + + depends_on = [ + null_resource.acm_operator + ] +} + +data "template_file" "acm_config" { + template = file("${path.module}/templates/acm-config.yml.tpl") + + vars = { + cluster_name = var.cluster_name + sync_repo = var.sync_repo + sync_branch = var.sync_branch + policy_dir = var.policy_dir + secret_type = "ssh" + } +} + +resource "null_resource" "acm_config" { + provisioner "local-exec" { + command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f -" + } + + provisioner "local-exec" { + when = "destroy" + command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f -" + } + + depends_on = [ + null_resource.acm_operator, + null_resource.git_creds_secret, + ] +} \ No newline at end of file diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf new file mode 100644 index 0000000000..b37d521c0e --- /dev/null +++ b/modules/acm/outputs.tf @@ -0,0 +1,4 @@ +output "git_creds_public" { + description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository." + value = tls_private_key.git_creds.public_key_openssh +} \ No newline at end of file diff --git a/modules/acm/scripts/kubectl_wrapper.sh b/modules/acm/scripts/kubectl_wrapper.sh new file mode 100755 index 0000000000..e92300bcb5 --- /dev/null +++ b/modules/acm/scripts/kubectl_wrapper.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -e + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +HOST=$1 +TOKEN=$2 +CA_CERTIFICATE=$3 + +shift 3 + +RANDOM_ID="${RANDOM}_${RANDOM}" +export TMPDIR="/tmp/kubectl_wrapper_${RANDOM_ID}" + +function cleanup { + rm -rf "${TMPDIR}" +} +trap cleanup EXIT + +mkdir "${TMPDIR}" + +export KUBECONFIG="${TMPDIR}/config" + +# shellcheck disable=SC1117 +base64 --help | grep "\--decode" && B64_ARG="--decode" || B64_ARG="-d" +echo "${CA_CERTIFICATE}" | base64 ${B64_ARG} > "${TMPDIR}/ca_certificate" + +kubectl config set-cluster kubectl-wrapper --server="${HOST}" --certificate-authority="${TMPDIR}/ca_certificate" --embed-certs=true 1>/dev/null +rm -f "${TMPDIR}/ca_certificate" +kubectl config set-context kubectl-wrapper --cluster=kubectl-wrapper --user=kubectl-wrapper --namespace=default 1>/dev/null +kubectl config set-credentials kubectl-wrapper --token="${TOKEN}" 1>/dev/null +kubectl config use-context kubectl-wrapper 1>/dev/null +kubectl version 1>/dev/null + +"$@" diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl new file mode 100644 index 0000000000..af60b8690a --- /dev/null +++ b/modules/acm/templates/acm-config.yml.tpl @@ -0,0 +1,12 @@ +apiVersion: configmanagement.gke.io/v1 +kind: ConfigManagement +metadata: + name: config-management +spec: + # clusterName is required and must be unique among all managed clusters + clusterName: ${cluster_name} + git: + syncRepo: ${sync_repo} + syncBranch: ${sync_branch} + secretType: ${secret_type} + policyDir: ${policy_dir} \ No newline at end of file diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf new file mode 100644 index 0000000000..9640135b24 --- /dev/null +++ b/modules/acm/variables.tf @@ -0,0 +1,30 @@ +variable "cluster_name" { + description = "The name of the cluster." + type = string +} + +variable "project_id" { + description = "The project in which the resource belongs." + type = string +} + +variable "location" { + description = "The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified." + type = string +} + +variable "sync_repo" { + description = "Anthos config management Git repo" + type = string +} + +variable "sync_branch" { + description = "Anthos config management Git branch" + type = string + default = "master" +} + +variable "policy_dir" { + description = "Subfolder containing configs in Ahtons config management Git repo" + type = string +} \ No newline at end of file diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 29facd32a9..fd8a62a18f 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -18,6 +18,7 @@ locals { int_required_roles = [ "roles/cloudkms.cryptoKeyEncrypterDecrypter", "roles/compute.networkAdmin", + "roles/container.admin", "roles/container.clusterAdmin", "roles/container.developer", "roles/iam.serviceAccountAdmin", diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh index 70ab3db5c8..ddedde4b4c 100755 --- a/test/task_helper_functions.sh +++ b/test/task_helper_functions.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env bash{ # Copyright 2019 Google LLC # From 2c3eb0c1baf2b9321b8e0352804843a11cae81cb Mon Sep 17 00:00:00 2001 From: pp Date: Thu, 24 Oct 2019 18:45:08 +0300 Subject: [PATCH 02/35] Fix destruction order * Add dependency on k8s endpoint * Fix lint-tests --- autogen/README.md | 2 +- examples/simple_zonal/acm.tf | 15 ++++++------ examples/simple_zonal/outputs.tf | 3 ++- examples/simple_zonal/variables.tf | 3 ++- modules/acm/README.md | 12 ++++++---- modules/acm/main.tf | 21 ++++++++++++++-- modules/acm/outputs.tf | 19 ++++++++++++++- modules/acm/templates/acm-config.yml.tpl | 3 ++- modules/acm/variables.tf | 24 ++++++++++++++++++- .../README.md | 2 +- .../main.tf | 3 ++- .../networks.tf | 2 +- .../variables.tf | 3 ++- .../private-cluster-update-variant/README.md | 2 +- .../private-cluster-update-variant/main.tf | 3 ++- .../networks.tf | 2 +- .../variables.tf | 3 ++- 17 files changed, 94 insertions(+), 28 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 421e4a2605..14d7409c7f 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -195,7 +195,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal/acm.tf index 1c38d22c99..47fdce1d86 100644 --- a/examples/simple_zonal/acm.tf +++ b/examples/simple_zonal/acm.tf @@ -15,11 +15,12 @@ */ module "acm" { - source = "../../modules/acm" - project_id = var.project_id - location = module.gke.location - cluster_name = module.gke.name - sync_repo = var.acm_sync_repo - sync_branch = var.acm_sync_branch - policy_dir = var.acm_policy_dir + source = "../../modules/acm" + project_id = var.project_id + location = module.gke.location + cluster_name = module.gke.name + sync_repo = var.acm_sync_repo + sync_branch = var.acm_sync_branch + policy_dir = var.acm_policy_dir + cluster_endpoint = module.gke.endpoint } diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf index 143710f8b2..2492208f7c 100644 --- a/examples/simple_zonal/outputs.tf +++ b/examples/simple_zonal/outputs.tf @@ -36,4 +36,5 @@ output "service_account" { output "acm_git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository." value = module.acm.git_creds_public -} \ No newline at end of file +} + diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal/variables.tf index adc92dba17..39a97b02d0 100644 --- a/examples/simple_zonal/variables.tf +++ b/examples/simple_zonal/variables.tf @@ -64,4 +64,5 @@ variable "acm_policy_dir" { description = "Subfolder containing configs in Ahtons config management Git repo" type = string default = "foo-corp" -} \ No newline at end of file +} + diff --git a/modules/acm/README.md b/modules/acm/README.md index 99a40bc840..40fdb939db 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -1,6 +1,6 @@ -# Terraform Kubernetes Engine ACM Submodule +# Terraform Kubernetes Engine ACM Submodule -This module installs Anthos Config Management (ACM) in a Kubernetes cluster. +This module installs Anthos Config Management (ACM) in a Kubernetes cluster. To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/). @@ -9,11 +9,11 @@ To find out more about ACM check [documentation](https://cloud.google.com/anthos In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md) plus **roles/container.admin** role. -## Usage example +## Usage example See [examples/simple_zonal](../../examples/simple_zonal) cluster example. -## Installation +## Installation Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page. Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public** @@ -25,6 +25,7 @@ output variable to your account at your Git server. | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes | | cluster\_name | The name of the cluster. | string | n/a | yes | | location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes | | policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes | @@ -38,4 +39,5 @@ output variable to your account at your Git server. |------|-------------| | git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. | - \ No newline at end of file + + diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 7a2a6019d4..10e55e9f88 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -1,5 +1,21 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + locals { - cluster_endpoint = "https://${data.google_container_cluster.primary.endpoint}" + cluster_endpoint = "https://${var.cluster_endpoint}" token = data.google_client_config.default.access_token cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate } @@ -87,4 +103,5 @@ resource "null_resource" "acm_config" { null_resource.acm_operator, null_resource.git_creds_secret, ] -} \ No newline at end of file +} + diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf index b37d521c0e..0ebca72e42 100644 --- a/modules/acm/outputs.tf +++ b/modules/acm/outputs.tf @@ -1,4 +1,21 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + output "git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository." value = tls_private_key.git_creds.public_key_openssh -} \ No newline at end of file +} + diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl index af60b8690a..8781f9db79 100644 --- a/modules/acm/templates/acm-config.yml.tpl +++ b/modules/acm/templates/acm-config.yml.tpl @@ -9,4 +9,5 @@ spec: syncRepo: ${sync_repo} syncBranch: ${sync_branch} secretType: ${secret_type} - policyDir: ${policy_dir} \ No newline at end of file + policyDir: ${policy_dir} + diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index 9640135b24..94b720e479 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -1,3 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + variable "cluster_name" { description = "The name of the cluster." type = string @@ -27,4 +43,10 @@ variable "sync_branch" { variable "policy_dir" { description = "Subfolder containing configs in Ahtons config management Git repo" type = string -} \ No newline at end of file +} + +variable "cluster_endpoint" { + description = "Kubernetes cluster endpoint." + type = string +} + diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 0fc0068e96..41279cc522 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -188,7 +188,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index fc38644871..63bf31ac78 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -23,7 +23,7 @@ data "google_compute_zones" "available" { provider = google-beta project = var.project_id - region = var.region + region = local.region } resource "random_shuffle" "available_zones" { @@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" { locals { // location location = var.regional ? var.region : var.zones[0] + region = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) // kuberentes version diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf index 14ea500e03..2456654130 100644 --- a/modules/beta-private-cluster-update-variant/networks.tf +++ b/modules/beta-private-cluster-update-variant/networks.tf @@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" { provider = google-beta name = var.subnetwork - region = var.region + region = local.region project = local.network_project_id } diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 9a869a830f..d783248ea2 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -40,7 +40,8 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null } variable "zones" { diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index e817361124..8b0a140c4c 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -174,7 +174,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index bfe746401c..2bd1c40d14 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -23,7 +23,7 @@ data "google_compute_zones" "available" { provider = google project = var.project_id - region = var.region + region = local.region } resource "random_shuffle" "available_zones" { @@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" { locals { // location location = var.regional ? var.region : var.zones[0] + region = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) // kuberentes version diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf index a382073dc0..aae034eee5 100644 --- a/modules/private-cluster-update-variant/networks.tf +++ b/modules/private-cluster-update-variant/networks.tf @@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" { provider = google name = var.subnetwork - region = var.region + region = local.region project = local.network_project_id } diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 8008e08975..28b744d868 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -40,7 +40,8 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null } variable "zones" { From 9720e805f6562bd931084e43439456ed4b8f36b2 Mon Sep 17 00:00:00 2001 From: pp Date: Tue, 29 Oct 2019 14:40:53 +0200 Subject: [PATCH 03/35] Minor grammatical corrections --- autogen/README.md | 96 ----------------------------------- modules/acm/README.md | 6 +-- test/task_helper_functions.sh | 2 +- 3 files changed, 3 insertions(+), 101 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 14d7409c7f..7b854e0351 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -139,102 +139,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/modules/acm/README.md b/modules/acm/README.md index 40fdb939db..8bc8abc9f0 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -15,10 +15,8 @@ See [examples/simple_zonal](../../examples/simple_zonal) cluster example. ## Installation -Module automate installation steps described at [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) page. -Git access provisioned via "Using an SSH keypair" option, to finish authorization add SSH public from **git\_creds\_public** -output variable to your account at your Git server. - +This module automates the instructions described in the [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) guide. +To enable Git access to the configuration repository over SSH, complete step 2 in the [Using an SSH keypair](https://cloud.google.com/anthos-config-management/docs/how-to/installing#git-creds-ssh) section using the SSH public key from the **git\_creds\_public** output. ## Inputs diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh index ddedde4b4c..70ab3db5c8 100755 --- a/test/task_helper_functions.sh +++ b/test/task_helper_functions.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash{ +#!/usr/bin/env bash # Copyright 2019 Google LLC # From 983952558b54c8d3f6c0c271517d330efaf04302 Mon Sep 17 00:00:00 2001 From: pp Date: Wed, 30 Oct 2019 18:58:52 +0200 Subject: [PATCH 04/35] Fix `gsutil` credentials --- modules/acm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 10e55e9f88..117e13c94b 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" { resource "null_resource" "acm_operator_config" { provisioner "local-exec" { - command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" + command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" } provisioner "local-exec" { From 6b13a01f16134872b4cadf42fa0df3f2c9e6457b Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 31 Oct 2019 18:38:33 -0500 Subject: [PATCH 05/35] add node pool location, enable and update test --- .kitchen.yml | 16 ++-- autogen/cluster.tf | 4 + examples/node_pool/main.tf | 17 +++- .../cluster.tf | 4 +- modules/beta-private-cluster/cluster.tf | 4 +- modules/beta-public-cluster/cluster.tf | 4 +- test/integration/node_pool/controls/gcloud.rb | 92 ++++++++++++++++++- 7 files changed, 126 insertions(+), 15 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 81603782cd..b5471f7e49 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -45,15 +45,13 @@ suites: systems: - name: disable_client_cert backend: local -# Disabled due to issue #274 -# (https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/274) -# - name: "node_pool" -# driver: -# root_module_directory: test/fixtures/node_pool -# verifier: -# systems: -# - name: node_pool -# backend: local + - name: "node_pool" + driver: + root_module_directory: test/fixtures/node_pool + verifier: + systems: + - name: node_pool + backend: local - name: "shared_vpc" driver: root_module_directory: test/fixtures/shared_vpc diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 0bac34a37e..2bd52acea6 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -310,6 +310,10 @@ resource "google_container_node_pool" "pools" { {% endif %} project = var.project_id location = local.location + {% if beta_cluster %} + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + {% endif %} cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index c7a7f852ae..3120c5735f 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -27,7 +27,6 @@ module "gke" { source = "../../modules/beta-public-cluster/" project_id = var.project_id name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = false region = var.region zones = var.zones network = var.network @@ -59,12 +58,24 @@ module "gke" { auto_repair = false service_account = var.compute_engine_service_account }, + { + name = "pool-03" + node_locations = "us-east4-b,us-east4-c" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 2 + disk_type = "pd-standard" + image_type = "COS" + auto_upgrade = true + service_account = var.compute_engine_service_account + }, ] node_pools_oauth_scopes = { all = [] pool-01 = [] pool-02 = [] + pool-03 = [] } node_pools_metadata = { @@ -73,6 +84,7 @@ module "gke" { shutdown-script = file("${path.module}/data/shutdown-script.sh") } pool-02 = {} + pool-03 = {} } node_pools_labels = { @@ -83,6 +95,7 @@ module "gke" { pool-01-example = true } pool-02 = {} + pool-03 = {} } node_pools_taints = { @@ -101,6 +114,7 @@ module "gke" { }, ] pool-02 = [] + pool-03 = [] } node_pools_tags = { @@ -111,6 +125,7 @@ module "gke" { "pool-01-example", ] pool-02 = [] + pool-03 = [] } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 6039cecd98..bc9d349205 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -284,7 +284,9 @@ resource "google_container_node_pool" "pools" { name = random_id.name.*.hex[count.index] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 10e12a9ba0..2aa7521203 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -212,7 +212,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index b5f896bc1b..24ec6f9b35 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -207,7 +207,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + // use node_locations if provided, defaults to cluster level node_locations if not specified + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 6ff5fdd201..170827f2d5 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -36,8 +36,8 @@ describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } - it "has 2" do - expect(node_pools.count).to eq 2 + it "has 3" do + expect(node_pools.count).to eq 3 end describe "pool-01" do @@ -279,6 +279,94 @@ ) end end + describe "pool-03" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + ) + ) + end + it "is the expected machine type" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has autoscaling enabled" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has autorepair enabled" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + + it "has automatic upgrades enabled" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "management" => including( + "autoUpgrade" => true, + ), + ) + ) + end + + end + end + end + describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + it "pool-03 has nodes in correct locations" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "locations" => match_array([ + "us-east4-b", + "us-east4-c", + ]), + ) + ) end end end From ecc434c9dcbd7af51faa7ceea8274c22eec38406 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 31 Oct 2019 19:15:18 -0500 Subject: [PATCH 06/35] update docs --- README.md | 75 ------------- autogen/README.md | 3 + .../README.md | 102 +----------------- modules/beta-private-cluster/README.md | 102 +----------------- modules/beta-public-cluster/README.md | 98 +---------------- .../private-cluster-update-variant/README.md | 79 -------------- modules/private-cluster/README.md | 79 -------------- 7 files changed, 6 insertions(+), 532 deletions(-) diff --git a/README.md b/README.md index 15f6aff13b..66623aa0c4 100644 --- a/README.md +++ b/README.md @@ -125,81 +125,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/autogen/README.md b/autogen/README.md index 846d339911..3efe785ff0 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -55,6 +55,9 @@ module "gke" { { name = "default-node-pool" machine_type = "n1-standard-2" + {% if beta_cluster %} + node_locations = "us-central1-b,us-central1-c" + {% endif %} min_count = 1 max_count = 100 disk_size_gb = 100 diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 93daea71ca..adba7958e7 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -48,6 +48,7 @@ module "gke" { { name = "default-node-pool" machine_type = "n1-standard-2" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 disk_size_gb = 100 @@ -132,107 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 4096848139..12189ad61d 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -48,6 +48,7 @@ module "gke" { { name = "default-node-pool" machine_type = "n1-standard-2" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 disk_size_gb = 100 @@ -132,107 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 258ea431e4..87a6b59e08 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -43,6 +43,7 @@ module "gke" { { name = "default-node-pool" machine_type = "n1-standard-2" + node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 disk_size_gb = 100 @@ -127,103 +128,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index fa9cdb8852..63c780a2d3 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 5465544b82..729bca2d46 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements From d87adebe64126763bb458ae62c400beb2e611e21 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 31 Oct 2019 19:18:50 -0500 Subject: [PATCH 07/35] fix docs --- README.md | 75 +++++++++++++ .../README.md | 101 ++++++++++++++++++ modules/beta-private-cluster/README.md | 101 ++++++++++++++++++ modules/beta-public-cluster/README.md | 97 +++++++++++++++++ .../private-cluster-update-variant/README.md | 79 ++++++++++++++ modules/private-cluster/README.md | 79 ++++++++++++++ 6 files changed, 532 insertions(+) diff --git a/README.md b/README.md index 66623aa0c4..15f6aff13b 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,81 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index adba7958e7..d825838b33 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -133,6 +133,107 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 12189ad61d..9f2d8a7ed9 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -133,6 +133,107 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 87a6b59e08..7f81e81500 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -128,6 +128,103 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 63c780a2d3..fa9cdb8852 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 729bca2d46..5465544b82 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements From 29c2e5978d592e4605e0d6bc8485ce20678f575d Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 31 Oct 2019 19:30:06 -0500 Subject: [PATCH 08/35] add test to ci --- build/int.cloudbuild.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 5a52a5889b..e58a1a66c0 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -41,6 +41,26 @@ steps: - verify disable-client-cert-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local'] +- id: create node-pool-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create node-pool-local'] +- id: converge node-pool-local + waitFor: + - create node-pool-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local'] +- id: verify node-pool-local + waitFor: + - converge node-pool-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local'] +- id: destroy node-pool-local + waitFor: + - verify node-pool-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local'] - id: create shared-vpc-local waitFor: - prepare From 647ffd334202239302e354eb8aea5c1cae6c22b1 Mon Sep 17 00:00:00 2001 From: pp Date: Fri, 1 Nov 2019 12:35:00 +0200 Subject: [PATCH 09/35] Removed `gsutil` credentials fix --- build/int.cloudbuild.yaml | 2 +- modules/acm/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 5a52a5889b..acc848d782 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -130,7 +130,7 @@ steps: waitFor: - create simple-zonal-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && env > /dev/null && kitchen_do converge simple-zonal-local'] - id: verify simple-zonal-local waitFor: - converge simple-zonal-local diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 117e13c94b..10e55e9f88 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" { resource "null_resource" "acm_operator_config" { provisioner "local-exec" { - command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" + command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" } provisioner "local-exec" { From f884041ec5c64d21a0ab766ca0a637e649b36ffb Mon Sep 17 00:00:00 2001 From: pp Date: Mon, 4 Nov 2019 17:12:23 +0200 Subject: [PATCH 10/35] Gsutil creds fix --- modules/acm/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 10e55e9f88..117e13c94b 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" { resource "null_resource" "acm_operator_config" { provisioner "local-exec" { - command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" + command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" } provisioner "local-exec" { From 763258511e05e66c859b04634f0a15b640642cc4 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 8 Nov 2019 11:06:58 -0600 Subject: [PATCH 11/35] fix yaml --- .kitchen.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index ff04c88dcf..88ca53374e 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -36,8 +36,8 @@ suites: systems: - name: disable_client_cert backend: local - controls: - - gcloud + controls: + - gcloud - name: "node_pool" driver: root_module_directory: test/fixtures/node_pool From a088b8f24292b8c3fffdfe440dea2d04e477dec4 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 8 Nov 2019 11:13:36 -0600 Subject: [PATCH 12/35] remove duplicate test suite with latest --- .kitchen.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 88ca53374e..39faa2e1e3 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -38,13 +38,6 @@ suites: backend: local controls: - gcloud - - name: "node_pool" - driver: - root_module_directory: test/fixtures/node_pool - verifier: - systems: - - name: node_pool - backend: local - name: "shared_vpc" driver: root_module_directory: test/fixtures/shared_vpc From a4bfc30169b1b156358751b0ef43882d3f5d4945 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 8 Nov 2019 11:19:52 -0600 Subject: [PATCH 13/35] fix cloudbuild --- build/int.cloudbuild.yaml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 6a9fcb2d35..09b5d3eb7e 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -41,26 +41,6 @@ steps: - verify disable-client-cert-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local'] -- id: create node-pool-local - waitFor: - - prepare - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create node-pool-local'] -- id: converge node-pool-local - waitFor: - - create node-pool-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local'] -- id: verify node-pool-local - waitFor: - - converge node-pool-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local'] -- id: destroy node-pool-local - waitFor: - - verify node-pool-local - name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local'] - id: create shared-vpc-local waitFor: - prepare From ef536c0c158f0612e3278c6532eecd4b46b8de4f Mon Sep 17 00:00:00 2001 From: "paul.p" Date: Fri, 8 Nov 2019 16:41:19 +0200 Subject: [PATCH 14/35] Remove "env" command form build/int.cloudbuild.yaml Co-Authored-By: Aaron Lane --- build/int.cloudbuild.yaml | 2 +- modules/acm/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 110f1171d4..09b5d3eb7e 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -130,7 +130,7 @@ steps: waitFor: - create simple-zonal-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && env > /dev/null && kitchen_do converge simple-zonal-local'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local'] - id: verify simple-zonal-local waitFor: - converge simple-zonal-local diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 117e13c94b..10e55e9f88 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -36,7 +36,7 @@ resource "tls_private_key" "git_creds" { resource "null_resource" "acm_operator_config" { provisioner "local-exec" { - command = "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=$GOOGLE_APPLICATION_CREDENTIALS gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" + command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" } provisioner "local-exec" { From 582236723b6b2895835d39e3b5fe77a7950859ed Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Fri, 8 Nov 2019 16:02:21 -0800 Subject: [PATCH 15/35] Fix node_count when autoscaling disabled on node pool. Fixes #311 Dont set initial_node_count when autoscaling is disabled on node pool. Use new node pool var when setting desired size of pool - matches provider var --- autogen/cluster.tf | 10 ++++++---- cluster.tf | 10 ++++++---- modules/beta-private-cluster-update-variant/cluster.tf | 10 ++++++---- modules/beta-private-cluster/cluster.tf | 10 ++++++---- modules/beta-public-cluster/cluster.tf | 10 ++++++---- modules/private-cluster-update-variant/cluster.tf | 10 ++++++---- modules/private-cluster/cluster.tf | 10 ++++++---- 7 files changed, 42 insertions(+), 28 deletions(-) diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 0bac34a37e..7353468cd7 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -316,16 +316,18 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + {% if beta_cluster %} max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) {% endif %} - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/cluster.tf b/cluster.tf index 7e5f53ee47..072a60fb14 100644 --- a/cluster.tf +++ b/cluster.tf @@ -137,13 +137,15 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 6039cecd98..c92695ce49 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -290,14 +290,16 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 10e12a9ba0..fb9b1ee683 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -218,14 +218,16 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index b5f896bc1b..88725475a8 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -213,14 +213,16 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + max_pods_per_node = lookup(var.node_pools[count.index], "max_pods_per_node", null) - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 615fe84bcc..d7fc2dd736 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -214,13 +214,15 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 3c42e64325..c8051255bf 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -142,13 +142,15 @@ resource "google_container_node_pool" "pools" { "version", local.node_version, ) - initial_node_count = lookup( + + initial_node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? lookup( var.node_pools[count.index], "initial_node_count", - lookup(var.node_pools[count.index], "min_count", 1), - ) + lookup(var.node_pools[count.index], "min_count", 1) + ) : null + - node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "min_count", 1) + node_count = lookup(var.node_pools[count.index], "autoscaling", true) ? null : lookup(var.node_pools[count.index], "node_count", 1) dynamic "autoscaling" { for_each = lookup(var.node_pools[count.index], "autoscaling", true) ? [var.node_pools[count.index]] : [] From 52cd66c49b178a8935e50ee8ae4d498cfe942745 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Fri, 8 Nov 2019 16:03:09 -0800 Subject: [PATCH 16/35] update test and example to show usage of a node pool with autoscaling off --- examples/node_pool/main.tf | 12 +++ test/integration/node_pool/controls/gcloud.rb | 97 ++++++++++++++++++- .../integration/node_pool/controls/kubectl.rb | 15 +++ 3 files changed, 122 insertions(+), 2 deletions(-) diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index c7a7f852ae..17b3bf8f67 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -59,12 +59,20 @@ module "gke" { auto_repair = false service_account = var.compute_engine_service_account }, + { + name = "pool-03" + autoscaling = false + node_count = 2 + service_account = var.compute_engine_service_account + auto_upgrade = true + }, ] node_pools_oauth_scopes = { all = [] pool-01 = [] pool-02 = [] + pool-03 = [] } node_pools_metadata = { @@ -73,6 +81,7 @@ module "gke" { shutdown-script = file("${path.module}/data/shutdown-script.sh") } pool-02 = {} + pool-03 = {} } node_pools_labels = { @@ -83,6 +92,7 @@ module "gke" { pool-01-example = true } pool-02 = {} + pool-03 = {} } node_pools_taints = { @@ -101,6 +111,7 @@ module "gke" { }, ] pool-02 = [] + pool-03 = [] } node_pools_tags = { @@ -111,6 +122,7 @@ module "gke" { "pool-01-example", ] pool-02 = [] + pool-03 = [] } } diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 6ff5fdd201..675a2e39ae 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -36,8 +36,8 @@ describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } - it "has 2" do - expect(node_pools.count).to eq 2 + it "has 3" do + expect(node_pools.count).to eq 3 end describe "pool-01" do @@ -279,6 +279,99 @@ ) end end + + describe "pool-03" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + ) + ) + end + + it "is the expected machine type" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has autoscaling disabled" do + expect(data['nodePools']).not_to include( + including( + "name" => "pool-03", + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected node count" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "initialNodeCount" => 2 + ) + ) + end + + it "has autorepair enabled" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + + it "has automatic upgrades enabled" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "management" => including( + "autoUpgrade" => true, + ), + ) + ) + end + + it "has the expected labels" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "config" => including( + "labels" => { + "all-pools-example" => "true", + "cluster_name" => cluster_name, + "node_pool" => "pool-03", + }, + ), + ) + ) + end + + it "has the expected network tags" do + expect(data['nodePools']).to include( + including( + "name" => "pool-03", + "config" => including( + "tags" => match_array([ + "all-node-example", + "gke-#{cluster_name}", + "gke-#{cluster_name}-pool-03", + ]), + ), + ) + ) + end + end end end end diff --git a/test/integration/node_pool/controls/kubectl.rb b/test/integration/node_pool/controls/kubectl.rb index 471f9cb33f..811ebcda0f 100644 --- a/test/integration/node_pool/controls/kubectl.rb +++ b/test/integration/node_pool/controls/kubectl.rb @@ -72,6 +72,21 @@ all_nodes.select { |n| n.metadata.labels.node_pool == "pool-02" } end + it "has the expected taints" do + expect(taints).to include( + { + effect: "PreferNoSchedule", + key: "all-pools-example", + value: "true", + } + ) + end + end + describe "pool-03" do + let(:nodes) do + all_nodes.select { |n| n.metadata.labels.node_pool == "pool-03" } + end + it "has the expected taints" do expect(taints).to include( { From e82a77e45a24a27d966ace400239d7b7bc64766e Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Fri, 8 Nov 2019 16:21:26 -0800 Subject: [PATCH 17/35] fix formatting --- examples/node_pool/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 17b3bf8f67..bf892be2a9 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -60,9 +60,9 @@ module "gke" { service_account = var.compute_engine_service_account }, { - name = "pool-03" - autoscaling = false - node_count = 2 + name = "pool-03" + autoscaling = false + node_count = 2 service_account = var.compute_engine_service_account auto_upgrade = true }, From 70688eeb611e4371030ee61b9d9115f0f0f3a21e Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Mon, 11 Nov 2019 11:23:52 -0800 Subject: [PATCH 18/35] force codebuild From ee90e456da14afb82e7147f50f789239a8468e03 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 11 Nov 2019 15:12:56 -0500 Subject: [PATCH 19/35] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a293e683..ba31673202 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Extending the adopted spec, each change should have a link to its corresponding ### Added * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300] +* Support for setting node_locations on node pools. [#303] ## [v5.1.1] - 2019-10-25 @@ -225,6 +226,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303 [#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284 From 48b94e8ff4f586ff113593e107fab328da536870 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 12 Nov 2019 14:33:23 -0800 Subject: [PATCH 20/35] add to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba31673202..226520c116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300] * Support for setting node_locations on node pools. [#303] +* Fix for specifying `node_count` on node pools when autoscaling is disabled. [#313] ## [v5.1.1] - 2019-10-25 From e3ae31ae1289754bc71916cbafacddd627885161 Mon Sep 17 00:00:00 2001 From: Taylor Ludwig Date: Tue, 12 Nov 2019 14:34:28 -0800 Subject: [PATCH 21/35] move node_pool example to us-central1 where there is cpu quota available after merging with master us-east4 maxes out default quota of 72 cpus in the region --- examples/node_pool/main.tf | 2 +- test/fixtures/shared/variables.tf | 5 ++--- test/integration/node_pool/controls/gcloud.rb | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index ad8d076f98..5bc0f53407 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -60,7 +60,7 @@ module "gke" { }, { name = "pool-03" - node_locations = "us-east4-b,us-east4-c" + node_locations = "${var.region}-b,${var.region}-c" autoscaling = false node_count = 2 machine_type = "n1-standard-2" diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index 5dff24dbd4..9760d65a94 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -20,13 +20,13 @@ variable "project_id" { variable "region" { description = "The GCP region to create and test resources in" - default = "us-east4" + default = "us-central1" } variable "zones" { type = list(string) description = "The GCP zones to create and test resources in, for applicable tests" - default = ["us-east4-a", "us-east4-b", "us-east4-c"] + default = ["us-central1-a", "us-central1-b", "us-central1-c"] } variable "compute_engine_service_account" { @@ -36,4 +36,3 @@ variable "compute_engine_service_account" { variable "registry_project_id" { description = "Project to use for granting access to the GCR registry, if requested" } - diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 2c6ed4f648..69a15e8293 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -392,8 +392,8 @@ including( "name" => "pool-03", "locations" => match_array([ - "us-east4-b", - "us-east4-c", + "us-central1-b", + "us-central1-c", ]), ) ) From edb49636f31e78e195f9fdd4c3ac577a1dea6bb0 Mon Sep 17 00:00:00 2001 From: pp Date: Wed, 13 Nov 2019 14:27:47 +0200 Subject: [PATCH 22/35] Updated docker image ver to fix gsutil cerds issue --- build/int.cloudbuild.yaml | 2 +- build/lint.cloudbuild.yaml | 2 +- test/setup/iam.tf | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 09b5d3eb7e..086203864d 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -306,6 +306,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 7ba0827bdb..02b9e5327b 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -24,4 +24,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.6' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' diff --git a/test/setup/iam.tf b/test/setup/iam.tf index fd8a62a18f..7ff4de74bc 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -25,7 +25,8 @@ locals { "roles/iam.serviceAccountUser", "roles/compute.networkAdmin", "roles/compute.viewer", - "roles/resourcemanager.projectIamAdmin" + "roles/resourcemanager.projectIamAdmin", + "roles/composer.worker" ] } From 67bd2c8436ae59995225ba5a4f870034cce1f4bf Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Wed, 13 Nov 2019 11:09:21 -0500 Subject: [PATCH 23/35] Add links for #300, #311 to CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 226520c116..e839926e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300] * Support for setting node_locations on node pools. [#303] -* Fix for specifying `node_count` on node pools when autoscaling is disabled. [#313] +* Fix for specifying `node_count` on node pools when autoscaling is disabled. [#311] ## [v5.1.1] - 2019-10-25 @@ -227,7 +227,9 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303 +[#300]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/300 [#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284 From e7851baeda5ba65167ddd960ba4cfdbdfec5bad5 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 15:27:10 -0500 Subject: [PATCH 24/35] Rename simple_zonal_with_acm example --- examples/{simple_zonal => simple_zonal_with_acm}/README.md | 6 ++++-- examples/{simple_zonal => simple_zonal_with_acm}/acm.tf | 0 examples/{simple_zonal => simple_zonal_with_acm}/main.tf | 0 examples/{simple_zonal => simple_zonal_with_acm}/outputs.tf | 0 .../{simple_zonal => simple_zonal_with_acm}/test_outputs.tf | 0 .../{simple_zonal => simple_zonal_with_acm}/variables.tf | 3 +-- test/fixtures/simple_zonal/example.tf | 3 +-- test/fixtures/simple_zonal/network.tf | 1 - 8 files changed, 6 insertions(+), 7 deletions(-) rename examples/{simple_zonal => simple_zonal_with_acm}/README.md (85%) rename examples/{simple_zonal => simple_zonal_with_acm}/acm.tf (100%) rename examples/{simple_zonal => simple_zonal_with_acm}/main.tf (100%) rename examples/{simple_zonal => simple_zonal_with_acm}/outputs.tf (100%) rename examples/{simple_zonal => simple_zonal_with_acm}/test_outputs.tf (100%) rename examples/{simple_zonal => simple_zonal_with_acm}/variables.tf (95%) diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal_with_acm/README.md similarity index 85% rename from examples/simple_zonal/README.md rename to examples/simple_zonal_with_acm/README.md index 2e514c6618..ea8539dc5f 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal_with_acm/README.md @@ -1,13 +1,15 @@ # Simple Zonal Cluster -This example illustrates how to create a simple cluster. +This example illustrates how to create a simple cluster and install [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/). + +It incorporates the standard cluster module and the [ACM install module](../../modules/acm). ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| acm\_policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | `"foo-corp"` | no | +| acm\_policy\_dir | Subfolder containing configs in ACM Git repo | string | `"foo-corp"` | no | | acm\_sync\_branch | Anthos config management Git branch | string | `"1.0.0"` | no | | acm\_sync\_repo | Anthos config management Git repo | string | `"git@github.com:GoogleCloudPlatform/csp-config-management.git"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | diff --git a/examples/simple_zonal/acm.tf b/examples/simple_zonal_with_acm/acm.tf similarity index 100% rename from examples/simple_zonal/acm.tf rename to examples/simple_zonal_with_acm/acm.tf diff --git a/examples/simple_zonal/main.tf b/examples/simple_zonal_with_acm/main.tf similarity index 100% rename from examples/simple_zonal/main.tf rename to examples/simple_zonal_with_acm/main.tf diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal_with_acm/outputs.tf similarity index 100% rename from examples/simple_zonal/outputs.tf rename to examples/simple_zonal_with_acm/outputs.tf diff --git a/examples/simple_zonal/test_outputs.tf b/examples/simple_zonal_with_acm/test_outputs.tf similarity index 100% rename from examples/simple_zonal/test_outputs.tf rename to examples/simple_zonal_with_acm/test_outputs.tf diff --git a/examples/simple_zonal/variables.tf b/examples/simple_zonal_with_acm/variables.tf similarity index 95% rename from examples/simple_zonal/variables.tf rename to examples/simple_zonal_with_acm/variables.tf index 39a97b02d0..2f427af33b 100644 --- a/examples/simple_zonal/variables.tf +++ b/examples/simple_zonal_with_acm/variables.tf @@ -61,8 +61,7 @@ variable "acm_sync_branch" { } variable "acm_policy_dir" { - description = "Subfolder containing configs in Ahtons config management Git repo" + description = "Subfolder containing configs in ACM Git repo" type = string default = "foo-corp" } - diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf index c0edb35e2b..4b76479dc5 100644 --- a/test/fixtures/simple_zonal/example.tf +++ b/test/fixtures/simple_zonal/example.tf @@ -15,7 +15,7 @@ */ module "example" { - source = "../../../examples/simple_zonal" + source = "../../../examples/simple_zonal_with_acm" project_id = var.project_id cluster_name_suffix = "-${random_string.suffix.result}" @@ -26,4 +26,3 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name } - diff --git a/test/fixtures/simple_zonal/network.tf b/test/fixtures/simple_zonal/network.tf index e1292eae3b..a4978c9ac3 100644 --- a/test/fixtures/simple_zonal/network.tf +++ b/test/fixtures/simple_zonal/network.tf @@ -45,4 +45,3 @@ resource "google_compute_subnetwork" "main" { ip_cidr_range = "192.168.64.0/18" } } - From 685b52593cfb04c3f373ef8bcc9d698141580bc3 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 15:56:55 -0500 Subject: [PATCH 25/35] Work on ACM docs --- modules/acm/README.md | 30 +++++++++++++++++++++++++++--- modules/acm/main.tf | 8 ++++---- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/modules/acm/README.md b/modules/acm/README.md index 8bc8abc9f0..7691fd6046 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -1,10 +1,34 @@ # Terraform Kubernetes Engine ACM Submodule -This module installs Anthos Config Management (ACM) in a Kubernetes cluster. -To find out more about ACM check [documentation](https://cloud.google.com/anthos-config-management/). +This module installs [Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/) (ACM) in a Kubernetes cluster. +Specifically, this module automates the following steps for [installing ACM](https://cloud.google.com/anthos-config-management/docs/how-to/installing): +1. Installing the ACM Operator on your cluster. +2. Generating an SSH key for accessing Git and providing it to the Operator +3. Configuring the Operator to connect to your ACM repository -## Configure a Service Account +## Usage + +There is a [full example](../../examples/simple_zonal) provided. Simple usage is as follows: + +```tf +module "acm" { + source = "../../modules/acm" + project_id = var.project_id + location = module.gke.location + cluster_name = module.gke.name + sync_repo = var.acm_sync_repo + sync_branch = var.acm_sync_branch + policy_dir = var.acm_policy_dir + cluster_endpoint = module.gke.endpoint +} +``` + + +In addition to this [example](../../examples/simple_zonal) shows how to provision a cluster and install ACm. + + +In order to use this module, you must use a Service Account In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md) plus **roles/container.admin** role. diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 10e55e9f88..1216c27e0f 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -40,7 +40,7 @@ resource "null_resource" "acm_operator_config" { } provisioner "local-exec" { - when = "destroy" + when = destroy command = "rm -f ${path.module}/config-management-operator.yaml" } } @@ -51,7 +51,7 @@ resource "null_resource" "acm_operator" { } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml" } @@ -68,7 +68,7 @@ resource "null_resource" "git_creds_secret" { } provisioner "local-exec" { - when = "destroy" + when = destroy command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete secret git-creds -n=config-management-system" } @@ -95,7 +95,7 @@ resource "null_resource" "acm_config" { } provisioner "local-exec" { - when = "destroy" + when = destroy command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f -" } From 018c6fb93dc90065e828cc066eedb7a35bd0fec1 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 16:55:39 -0500 Subject: [PATCH 26/35] Make private clusters expose public endpoint if var.deploy_using_private_endpoint is false --- autogen/main.tf | 2 +- modules/beta-private-cluster-update-variant/main.tf | 2 +- modules/beta-private-cluster/main.tf | 2 +- modules/private-cluster-update-variant/main.tf | 2 +- modules/private-cluster/main.tf | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/autogen/main.tf b/autogen/main.tf index 30347b9b15..841444ea44 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -92,7 +92,7 @@ locals { cluster_output_zones = local.cluster_output_regional_zones {% if private_cluster %} - cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint {% else %} cluster_output_endpoint = google_container_cluster.primary.endpoint {% endif %} diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 2de95c063d..5b235ce00f 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -83,7 +83,7 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 2de95c063d..5b235ce00f 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -83,7 +83,7 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index aba5e2d79f..f0d307311c 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -70,7 +70,7 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index aba5e2d79f..f0d307311c 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -70,7 +70,7 @@ locals { cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones - cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.endpoint + cluster_output_endpoint = var.deploy_using_private_endpoint ? google_container_cluster.primary.private_cluster_config.0.private_endpoint : google_container_cluster.primary.private_cluster_config.0.public_endpoint cluster_output_master_auth = concat(google_container_cluster.primary.*.master_auth, []) cluster_output_master_version = google_container_cluster.primary.master_version From b5ebae53357b29a171f8136796a5c152166c59e9 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 17:46:26 -0500 Subject: [PATCH 27/35] Make key generation optional for ACM module --- modules/acm/main.tf | 12 ++++++++++-- modules/acm/outputs.tf | 2 +- modules/acm/variables.tf | 5 +++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 1216c27e0f..439cb02fda 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -18,6 +18,7 @@ locals { cluster_endpoint = "https://${var.cluster_endpoint}" token = data.google_client_config.default.access_token cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate + private_key = var.create_ssh_key ? tls_private_key.git_creds[0].private_key_pem : "" } data "google_container_cluster" "primary" { @@ -30,6 +31,7 @@ data "google_client_config" "default" { } resource "tls_private_key" "git_creds" { + count = var.create_ssh_key ? 1 : 0 algorithm = "RSA" rsa_bits = 4096 } @@ -63,8 +65,10 @@ resource "null_resource" "acm_operator" { } resource "null_resource" "git_creds_secret" { + count = var.create_ssh_key ? 1 : 0 + provisioner "local-exec" { - command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${tls_private_key.git_creds.private_key_pem}'" + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl create secret generic git-creds -n=config-management-system --from-literal=ssh='${local.private_key}'" } provisioner "local-exec" { @@ -85,11 +89,15 @@ data "template_file" "acm_config" { sync_repo = var.sync_repo sync_branch = var.sync_branch policy_dir = var.policy_dir - secret_type = "ssh" + secret_type = var.create_ssh_key ? "ssh" : "none" } } resource "null_resource" "acm_config" { + triggers = { + config = data.template_file.acm_config.rendered + } + provisioner "local-exec" { command = "echo '${data.template_file.acm_config.rendered}' | ${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f -" } diff --git a/modules/acm/outputs.tf b/modules/acm/outputs.tf index 0ebca72e42..1945ba4f59 100644 --- a/modules/acm/outputs.tf +++ b/modules/acm/outputs.tf @@ -16,6 +16,6 @@ output "git_creds_public" { description = "Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository." - value = tls_private_key.git_creds.public_key_openssh + value = var.create_ssh_key ? tls_private_key.git_creds.*.public_key_openssh : null } diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index 94b720e479..faa91b201a 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -50,3 +50,8 @@ variable "cluster_endpoint" { type = string } +variable "create_ssh_key" { + description = "Controls whether a key will be generated for Git authentication" + type = bool + default = true +} From 0ee7ca4966d98f6ef87f16b2c116f02d68583eec Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 17:53:47 -0500 Subject: [PATCH 28/35] Clean up ACM variable descriptions --- modules/acm/README.md | 11 ++++++----- modules/acm/variables.tf | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/modules/acm/README.md b/modules/acm/README.md index 7691fd6046..9beb6ee270 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -48,12 +48,13 @@ To enable Git access to the configuration repository over SSH, complete step 2 i | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes | -| cluster\_name | The name of the cluster. | string | n/a | yes | -| location | The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified. | string | n/a | yes | -| policy\_dir | Subfolder containing configs in Ahtons config management Git repo | string | n/a | yes | +| cluster\_name | The unique name to identify the cluster by in ACM. | string | n/a | yes | +| create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no | +| location | The location (zone or region) this cluster has been created in. | string | n/a | yes | +| policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes | | project\_id | The project in which the resource belongs. | string | n/a | yes | -| sync\_branch | Anthos config management Git branch | string | `"master"` | no | -| sync\_repo | Anthos config management Git repo | string | n/a | yes | +| sync\_branch | ACM repo Git branch | string | `"master"` | no | +| sync\_repo | ACM Git repo address | string | n/a | yes | ## Outputs diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index faa91b201a..164e14d846 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -15,7 +15,7 @@ */ variable "cluster_name" { - description = "The name of the cluster." + description = "The unique name to identify the cluster in ACM." type = string } @@ -25,23 +25,23 @@ variable "project_id" { } variable "location" { - description = "The location (zone or region) this cluster has been created in. One of location, region, zone, or a provider-level zone must be specified." + description = "The location (zone or region) this cluster has been created in." type = string } variable "sync_repo" { - description = "Anthos config management Git repo" + description = "ACM Git repo address" type = string } variable "sync_branch" { - description = "Anthos config management Git branch" + description = "ACM repo Git branch" type = string default = "master" } variable "policy_dir" { - description = "Subfolder containing configs in Ahtons config management Git repo" + description = "Subfolder containing configs in ACM Git repo" type = string } From 8d174c6cc1974349ab434ef9a92d34749f2fe129 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 18:23:28 -0500 Subject: [PATCH 29/35] Add test for InSpec operator --- .kitchen.yml | 1 + test/integration/simple_zonal/controls/acm.rb | 52 +++++++++++++++++++ test/integration/simple_zonal/inspec.yml | 4 +- 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 test/integration/simple_zonal/controls/acm.rb diff --git a/.kitchen.yml b/.kitchen.yml index 39faa2e1e3..c0d08c2418 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -85,6 +85,7 @@ suites: backend: local controls: - gcloud + - acm - name: gcp backend: gcp controls: diff --git a/test/integration/simple_zonal/controls/acm.rb b/test/integration/simple_zonal/controls/acm.rb new file mode 100644 index 0000000000..45663f176d --- /dev/null +++ b/test/integration/simple_zonal/controls/acm.rb @@ -0,0 +1,52 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require 'kubeclient' +require 'rest-client' + +require 'base64' + +kubernetes_endpoint = attribute('kubernetes_endpoint') +client_token = attribute('client_token') +ca_certificate = attribute('ca_certificate') + +control "acm" do + title "Anthos Config Management" + + describe "kubernetes" do + let(:kubernetes_http_endpoint) { "https://#{kubernetes_endpoint}/api" } + let(:client) do + cert_store = OpenSSL::X509::Store.new + cert_store.add_cert(OpenSSL::X509::Certificate.new(Base64.decode64(ca_certificate))) + Kubeclient::Client.new( + kubernetes_http_endpoint, + "v1", + ssl_options: { + cert_store: cert_store, + verify_ssl: OpenSSL::SSL::VERIFY_PEER, + }, + auth_options: { + bearer_token: Base64.decode64(client_token), + }, + ) + end + + describe "config-management-system namespace" do + let(:namespace) { client.get_namespace("config-management-system") } + it "should exist" do + expect(namespace).not_to be nil + end + end + end +end diff --git a/test/integration/simple_zonal/inspec.yml b/test/integration/simple_zonal/inspec.yml index 3a3bc74145..4e41f27bad 100644 --- a/test/integration/simple_zonal/inspec.yml +++ b/test/integration/simple_zonal/inspec.yml @@ -25,4 +25,6 @@ attributes: - name: service_account required: true type: string - + - name: ca_certificate + required: true + type: string From 80b5704986510726d78439752ca37e971dc70328 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 19:15:18 -0500 Subject: [PATCH 30/35] Work on downloading operator in CI --- build/int.cloudbuild.yaml | 3 ++ examples/simple_zonal_with_acm/acm.tf | 1 + examples/simple_zonal_with_acm/variables.tf | 6 ++++ modules/acm/README.md | 38 +++++++++++---------- modules/acm/main.tf | 8 +++-- modules/acm/variables.tf | 6 ++++ test/fixtures/simple_zonal/example.tf | 1 + test/task_helper_functions.sh | 5 +++ 8 files changed, 48 insertions(+), 20 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 086203864d..e78d0eb2ba 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -14,6 +14,9 @@ timeout: 12600s steps: +- id: download acm + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && download_acm'] - id: prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment'] diff --git a/examples/simple_zonal_with_acm/acm.tf b/examples/simple_zonal_with_acm/acm.tf index 47fdce1d86..0af8f08b5d 100644 --- a/examples/simple_zonal_with_acm/acm.tf +++ b/examples/simple_zonal_with_acm/acm.tf @@ -23,4 +23,5 @@ module "acm" { sync_branch = var.acm_sync_branch policy_dir = var.acm_policy_dir cluster_endpoint = module.gke.endpoint + operator_path = var.operator_path } diff --git a/examples/simple_zonal_with_acm/variables.tf b/examples/simple_zonal_with_acm/variables.tf index 2f427af33b..4d7fce1d3a 100644 --- a/examples/simple_zonal_with_acm/variables.tf +++ b/examples/simple_zonal_with_acm/variables.tf @@ -65,3 +65,9 @@ variable "acm_policy_dir" { type = string default = "foo-corp" } + +variable "operator_path" { + description = "Path to the operator yaml config. If unset, will download from GCS releases." + type = string + default = null +} diff --git a/modules/acm/README.md b/modules/acm/README.md index 9beb6ee270..1dabfc2ddb 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -13,29 +13,31 @@ There is a [full example](../../examples/simple_zonal) provided. Simple usage is ```tf module "acm" { - source = "../../modules/acm" - project_id = var.project_id + source = "terraform-google-modules/kubernetes-engine/google//modules/acm" + + project_id = "my-project-id" + cluster_name = "my-cluster-name" location = module.gke.location - cluster_name = module.gke.name - sync_repo = var.acm_sync_repo - sync_branch = var.acm_sync_branch - policy_dir = var.acm_policy_dir cluster_endpoint = module.gke.endpoint + + sync_repo = "git@github.com:GoogleCloudPlatform/csp-config-management.git" + sync_branch = "1.0.0" + policy_dir = "foo-corp" } ``` - -In addition to this [example](../../examples/simple_zonal) shows how to provision a cluster and install ACm. - - -In order to use this module, you must use a Service Account - -In order to use this module you must have Service Account with roles listed [Terraform Kubernetes Engine Module](../../README.md) -plus **roles/container.admin** role. - -## Usage example - -See [examples/simple_zonal](../../examples/simple_zonal) cluster example. +To deploy this config: +1. Run `terraform apply` +2. Inspect the `git\_creds\_public` [output][#outputs] to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience: + + * [{{source_repos_name}}](/source-repositories/docs/authentication#ssh) + * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html){:.external} + * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/){:.external}. + * [Gitlab](https://docs.gitlab.com/ee/ssh/){:.external} + +## Whitelisting +Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license. +By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location. ## Installation diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 439cb02fda..d93dbdcdf3 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -19,6 +19,8 @@ locals { token = data.google_client_config.default.access_token cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate private_key = var.create_ssh_key ? tls_private_key.git_creds[0].private_key_pem : "" + download_operator = var.operator_path == null ? true : false + operator_path = local.download_operator ? "${path.module}/config-management-operator.yaml" : var.operator_path } data "google_container_cluster" "primary" { @@ -37,6 +39,8 @@ resource "tls_private_key" "git_creds" { } resource "null_resource" "acm_operator_config" { + count = local.download_operator ? 1 : 0 + provisioner "local-exec" { command = "gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml ${path.module}/config-management-operator.yaml" } @@ -49,12 +53,12 @@ resource "null_resource" "acm_operator_config" { resource "null_resource" "acm_operator" { provisioner "local-exec" { - command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${path.module}/config-management-operator.yaml" + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl apply -f ${local.operator_path}" } provisioner "local-exec" { when = destroy - command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${path.module}/config-management-operator.yaml" + command = "${path.module}/scripts/kubectl_wrapper.sh ${local.cluster_endpoint} ${local.token} ${local.cluster_ca_certificate} kubectl delete -f ${local.operator_path}" } depends_on = [ diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index 164e14d846..cc49006ae9 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -29,6 +29,12 @@ variable "location" { type = string } +variable "operator_path" { + description = "Path to the operator yaml config. If unset, will download from GCS releases." + type = string + default = null +} + variable "sync_repo" { description = "ACM Git repo address" type = string diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf index 4b76479dc5..7d558e36f0 100644 --- a/test/fixtures/simple_zonal/example.tf +++ b/test/fixtures/simple_zonal/example.tf @@ -25,4 +25,5 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + operator_path = "/workspace/acm.yaml" } diff --git a/test/task_helper_functions.sh b/test/task_helper_functions.sh index 7de28bbdfe..a4bac86bf6 100755 --- a/test/task_helper_functions.sh +++ b/test/task_helper_functions.sh @@ -50,3 +50,8 @@ function check_generate() { rm -Rf "${tempdir}" return $((rval)) } + +# Pre-download the Anthos Config Management operator +function download_acm() { + gsutil cp gs://config-management-release/released/latest/config-management-operator.yaml /workspace/acm.yaml +} From 7e4fe424bba89ec51d4df0e3f336e126f358d130 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 19:24:16 -0500 Subject: [PATCH 31/35] ACM docs cleanup --- examples/simple_zonal_with_acm/README.md | 1 + modules/acm/README.md | 18 +++++++----------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/examples/simple_zonal_with_acm/README.md b/examples/simple_zonal_with_acm/README.md index ea8539dc5f..b5def60e96 100644 --- a/examples/simple_zonal_with_acm/README.md +++ b/examples/simple_zonal_with_acm/README.md @@ -16,6 +16,7 @@ It incorporates the standard cluster module and the [ACM install module](../../m | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | network | The VPC network to host the cluster in | string | n/a | yes | +| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | diff --git a/modules/acm/README.md b/modules/acm/README.md index 1dabfc2ddb..bb05629898 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -28,31 +28,27 @@ module "acm" { To deploy this config: 1. Run `terraform apply` -2. Inspect the `git\_creds\_public` [output][#outputs] to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience: +2. Inspect the `git_creds_public` [output](#outputs) to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience: - * [{{source_repos_name}}](/source-repositories/docs/authentication#ssh) - * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html){:.external} - * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/){:.external}. - * [Gitlab](https://docs.gitlab.com/ee/ssh/){:.external} + * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh) + * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html) + * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/) + * [Gitlab](https://docs.gitlab.com/ee/ssh/) ## Whitelisting Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license. By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location. -## Installation - -This module automates the instructions described in the [Installing Anthos Config Management](https://cloud.google.com/anthos-config-management/docs/how-to/installing) guide. -To enable Git access to the configuration repository over SSH, complete step 2 in the [Using an SSH keypair](https://cloud.google.com/anthos-config-management/docs/how-to/installing#git-creds-ssh) section using the SSH public key from the **git\_creds\_public** output. - ## Inputs | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes | -| cluster\_name | The unique name to identify the cluster by in ACM. | string | n/a | yes | +| cluster\_name | The unique name to identify the cluster in ACM. | string | n/a | yes | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no | | location | The location (zone or region) this cluster has been created in. | string | n/a | yes | +| operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no | | policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes | | project\_id | The project in which the resource belongs. | string | n/a | yes | | sync\_branch | ACM repo Git branch | string | `"master"` | no | From f8d02c199e09e744ed1fa666f56bff1c8326f8ad Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 19:29:18 -0500 Subject: [PATCH 32/35] Add Policy Controller options to ACM installer --- modules/acm/README.md | 13 +++++++------ modules/acm/main.tf | 12 +++++++----- modules/acm/templates/acm-config.yml.tpl | 4 +++- modules/acm/variables.tf | 12 ++++++++++++ 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/modules/acm/README.md b/modules/acm/README.md index bb05629898..166611e0b7 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -30,11 +30,11 @@ To deploy this config: 1. Run `terraform apply` 2. Inspect the `git_creds_public` [output](#outputs) to retrieve the public key used for accessing Git. Whitelist this key for access to your Git repo. Instructions for some popular Git hosting providers are included for convenience: - * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh) - * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html) - * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/) - * [Gitlab](https://docs.gitlab.com/ee/ssh/) - + * [Cloud Souce Repositories](https://cloud.google.com/source-repositories/docs/authentication#ssh) + * [Bitbucket](https://confluence.atlassian.com/bitbucket/set-up-an-ssh-key-728138079.html) + * [GitHub](https://help.github.com/articles/adding-a-new-ssh-key-to-your-github-account/) + * [Gitlab](https://docs.gitlab.com/ee/ssh/) + ## Whitelisting Note that installing Anthos Config Management [requires](https://cloud.google.com/anthos-config-management/docs/how-to/installing#local_environment) an active Anthos license. By default, this module will attempt to download the ACM operator from Google directly—meaning your Terraform service account needs to be whitelisted for ACM access. If this is an issue, you can predownload the operator yourself then set the `operator_path` variable to point to the file location. @@ -47,6 +47,8 @@ By default, this module will attempt to download the ACM operator from Google di | cluster\_endpoint | Kubernetes cluster endpoint. | string | n/a | yes | | cluster\_name | The unique name to identify the cluster in ACM. | string | n/a | yes | | create\_ssh\_key | Controls whether a key will be generated for Git authentication | bool | `"true"` | no | +| enable\_policy\_controller | Whether to enable the ACM Policy Controller on the cluster | bool | `"true"` | no | +| install\_template\_library | Whether to install the default Policy Controller template library | bool | `"true"` | no | | location | The location (zone or region) this cluster has been created in. | string | n/a | yes | | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no | | policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes | @@ -61,4 +63,3 @@ By default, this module will attempt to download the ACM operator from Google di | git\_creds\_public | Public key of SSH keypair to allow the Anthos Operator to authenticate to your Git repository. | - diff --git a/modules/acm/main.tf b/modules/acm/main.tf index d93dbdcdf3..3a78172ad6 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -89,11 +89,13 @@ data "template_file" "acm_config" { template = file("${path.module}/templates/acm-config.yml.tpl") vars = { - cluster_name = var.cluster_name - sync_repo = var.sync_repo - sync_branch = var.sync_branch - policy_dir = var.policy_dir - secret_type = var.create_ssh_key ? "ssh" : "none" + cluster_name = var.cluster_name + sync_repo = var.sync_repo + sync_branch = var.sync_branch + policy_dir = var.policy_dir + secret_type = var.create_ssh_key ? "ssh" : "none" + enable_policy_controller = var.enable_policy_controller ? "true" : "false" + install_template_library = var.install_template_library ? "true" : "false" } } diff --git a/modules/acm/templates/acm-config.yml.tpl b/modules/acm/templates/acm-config.yml.tpl index 8781f9db79..80155737e8 100644 --- a/modules/acm/templates/acm-config.yml.tpl +++ b/modules/acm/templates/acm-config.yml.tpl @@ -10,4 +10,6 @@ spec: syncBranch: ${sync_branch} secretType: ${secret_type} policyDir: ${policy_dir} - + policyController: + enabled: ${enable_policy_controller} + templateLibraryInstalled: ${install_template_library} diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index cc49006ae9..513556364d 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -61,3 +61,15 @@ variable "create_ssh_key" { type = bool default = true } + +variable "enable_policy_controller" { + description = "Whether to enable the ACM Policy Controller on the cluster" + type = bool + default = true +} + +variable "install_template_library" { + description = "Whether to install the default Policy Controller template library" + type = bool + default = true +} From 3162c33ec84d31490acf87d72337d9e5bbf35764 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 15 Nov 2019 09:57:05 -0500 Subject: [PATCH 33/35] Add #268 to changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e839926e6b..e7e738732c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300] * Support for setting node_locations on node pools. [#303] * Fix for specifying `node_count` on node pools when autoscaling is disabled. [#311] +* Added submodule for installing Anthos Config Management. [#268] ## [v5.1.1] - 2019-10-25 @@ -227,6 +228,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268 [#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303 [#300]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/300 From 8dfd27b79700b941b9904bf97d0fe59260f112f8 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 15 Nov 2019 11:20:19 -0500 Subject: [PATCH 34/35] Update README.md --- modules/acm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/acm/README.md b/modules/acm/README.md index 166611e0b7..bef7eccd53 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -9,7 +9,7 @@ Specifically, this module automates the following steps for [installing ACM](htt ## Usage -There is a [full example](../../examples/simple_zonal) provided. Simple usage is as follows: +There is a [full example](../../examples/simple_zonal_with_acm) provided. Simple usage is as follows: ```tf module "acm" { From efa4b61e60fe8e2bb275add05b6542dfb43cc279 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Fri, 15 Nov 2019 11:31:11 -0500 Subject: [PATCH 35/35] Don't attempt to set node_locations on pools if unspecified --- autogen/cluster.tf | 2 +- modules/beta-private-cluster-update-variant/cluster.tf | 2 +- modules/beta-private-cluster/cluster.tf | 2 +- modules/beta-public-cluster/cluster.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 68e23ebabc..4791220591 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -312,7 +312,7 @@ resource "google_container_node_pool" "pools" { location = local.location {% if beta_cluster %} // use node_locations if provided, defaults to cluster level node_locations if not specified - node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null {% endif %} cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index a0b7f50b8d..10c206f35c 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -285,7 +285,7 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified - node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 042a4735cd..c363dacf9f 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -213,7 +213,7 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified - node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 2b4fb3a6e9..1f5eee84aa 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -208,7 +208,7 @@ resource "google_container_node_pool" "pools" { project = var.project_id location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified - node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : google_container_cluster.primary.node_locations + node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null cluster = google_container_cluster.primary.name version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index],