From e8688fce06b3a2aedb7f9df34b96d9f2eae4e7c5 Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 001/110] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..ac5726feeb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -61,6 +61,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From 599dd2e3d801a116c8d44c2633c3425ff0d0b78b Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 002/110] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 9f5df5a03e..ac5726feeb 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -61,6 +61,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From d969d8667902a47bd8129915502d8b873febaf67 Mon Sep 17 00:00:00 2001 From: Sam Gendler Date: Wed, 6 Nov 2019 00:45:16 -0800 Subject: [PATCH 003/110] add private_zonal_with_networking example --- .kitchen.yml | 17 ++ .../private_zonal_with_networking/main.tf | 82 ++++++++ .../private_zonal_with_networking/outputs.tf | 60 ++++++ .../test_outputs.tf | 58 ++++++ .../variables.tf | 54 ++++++ test/ci/private-zonal-with-networking.yml | 18 ++ .../private_zonal_with_networking/example.tf | 23 +++ .../private_zonal_with_networking/outputs.tf | 73 +++++++ .../variables.tf | 31 +++ .../controls/gcloud.rb | 180 ++++++++++++++++++ .../controls/network.rb | 28 +++ .../controls/subnet.rb | 46 +++++ .../private_zonal_with_networking/inspec.yml | 36 ++++ 13 files changed, 706 insertions(+) create mode 100644 examples/private_zonal_with_networking/main.tf create mode 100644 examples/private_zonal_with_networking/outputs.tf create mode 100644 examples/private_zonal_with_networking/test_outputs.tf create mode 100644 examples/private_zonal_with_networking/variables.tf create mode 100644 test/ci/private-zonal-with-networking.yml create mode 100644 test/fixtures/private_zonal_with_networking/example.tf create mode 100644 test/fixtures/private_zonal_with_networking/outputs.tf create mode 100644 test/fixtures/private_zonal_with_networking/variables.tf create mode 100644 test/integration/private_zonal_with_networking/controls/gcloud.rb create mode 100644 test/integration/private_zonal_with_networking/controls/network.rb create mode 100644 test/integration/private_zonal_with_networking/controls/subnet.rb create mode 100644 test/integration/private_zonal_with_networking/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 81603782cd..7707328091 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -68,6 +68,23 @@ suites: systems: - name: simple_regional backend: local + - name: "private_zonal_with_networking" + driver: + root_module_directory: test/fixtures/private_zonal_with_networking + verifier: + systems: + - name: private_zonal_with_networking + backend: local + controls: + - gcloud + - name: subnet + backend: local + controls: + - subnet + - name: network + backend: gcp + controls: + - network - name: "simple_regional_with_networking" driver: root_module_directory: test/fixtures/simple_regional_with_networking diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf new file mode 100644 index 0000000000..fb5d34d757 --- /dev/null +++ b/examples/private_zonal_with_networking/main.tf @@ -0,0 +1,82 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 1.4.0" + project_id = var.project_id + network_name = var.network + + subnets = [ + { + subnet_name = var.subnetwork + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = "true" + }, + ] + + secondary_ranges = { + "${var.subnetwork}" = [ + { + range_name = var.ip_range_pods_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = var.ip_range_services_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} + +data "google_compute_subnetwork" "subnetwork" { + name = module.gcp-network.subnets_names[0] + project = var.project_id + region = var.region + depends_on = [module.gcp-network] +} + +module "gke" { + source = "../../modules/beta-private-cluster/" + project_id = var.project_id + name = var.cluster_name + regional = false + region = var.region + zones = slice(var.zones, 0, 1) + network = data.google_compute_subnetwork.subnetwork.network + subnetwork = data.google_compute_subnetwork.subnetwork.name + ip_range_pods = var.ip_range_pods_name + ip_range_services = var.ip_range_services_name + create_service_account = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] +} + +data "google_client_config" "default" { +} diff --git a/examples/private_zonal_with_networking/outputs.tf b/examples/private_zonal_with_networking/outputs.tf new file mode 100644 index 0000000000..bb255b54a2 --- /dev/null +++ b/examples/private_zonal_with_networking/outputs.tf @@ -0,0 +1,60 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_name" { + description = "The name of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "subnet_secondary_ranges" { + description = "The secondary ranges associated with the subnet" + value = module.gcp-network.subnets_secondary_ranges +} + + + diff --git a/examples/private_zonal_with_networking/test_outputs.tf b/examples/private_zonal_with_networking/test_outputs.tf new file mode 100644 index 0000000000..a703679105 --- /dev/null +++ b/examples/private_zonal_with_networking/test_outputs.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.gke.region +} + +output "network" { + value = var.network +} + +output "subnetwork" { + value = var.subnetwork +} + +output "location" { + value = module.gke.location +} + +output "ip_range_pods_name" { + description = "The secondary IP range used for pods" + value = var.ip_range_pods_name +} + +output "ip_range_services_name" { + description = "The secondary IP range used for services" + value = var.ip_range_services_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.gke.master_version +} diff --git a/examples/private_zonal_with_networking/variables.tf b/examples/private_zonal_with_networking/variables.tf new file mode 100644 index 0000000000..ac2c68d2b6 --- /dev/null +++ b/examples/private_zonal_with_networking/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name" { + description = "The name for the GKE cluster" + default = "gke-on-vpc-cluster" +} + +variable "region" { + description = "The region to host the cluster in" + default = "us-central1" +} + +variable "network" { + description = "The VPC network created to host the cluster in" + default = "gke-network" +} + +variable "subnetwork" { + description = "The subnetwork created to host the cluster in" + default = "gke-subnet" +} + +variable "ip_range_pods_name" { + description = "The secondary ip range to use for pods" + default = "ip-range-pods" +} + +variable "ip_range_services_name" { + description = "The secondary ip range to use for pods" + default = "ip-range-scv" +} + +variable "zones" { + default = [] +} + diff --git a/test/ci/private-zonal-with-networking.yml b/test/ci/private-zonal-with-networking.yml new file mode 100644 index 0000000000..f628957a28 --- /dev/null +++ b/test/ci/private-zonal-with-networking.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "private-zonal-with-networking-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/private_zonal_with_networking/example.tf b/test/fixtures/private_zonal_with_networking/example.tf new file mode 100644 index 0000000000..23612f1610 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/example.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/private_zonal_with_networking" + + project_id = var.project_id + region = var.region + zones = var.zones +} diff --git a/test/fixtures/private_zonal_with_networking/outputs.tf b/test/fixtures/private_zonal_with_networking/outputs.tf new file mode 100644 index 0000000000..08f9a8a2e8 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/outputs.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +output "project_id" { + value = var.project_id +} + +output "location" { + value = module.example.location +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + value = module.example.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.example.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.example.network +} + +output "subnet_name" { + description = "The name of the subnet being created" + value = module.example.subnetwork +} + +output "region" { + description = "The region the cluster is hosted in" + value = module.example.region +} + +output "ip_range_pods_name" { + description = "The secondary range name for pods" + value = module.example.ip_range_pods_name +} + +output "ip_range_services_name" { + description = "The secondary range name for services" + value = module.example.ip_range_services_name +} diff --git a/test/fixtures/private_zonal_with_networking/variables.tf b/test/fixtures/private_zonal_with_networking/variables.tf new file mode 100644 index 0000000000..d610f1e807 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "region" { + description = "The region to host the cluster in" + default = "us-east4" +} + +variable "zones" { + type = list(string) + description = "The GCP zones to create and test resources in, for applicable tests" + default = ["us-east4-a", "us-east4-b", "us-east4-c"] +} + diff --git a/test/integration/private_zonal_with_networking/controls/gcloud.rb b/test/integration/private_zonal_with_networking/controls/gcloud.rb new file mode 100644 index 0000000000..e998182da2 --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/gcloud.rb @@ -0,0 +1,180 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses the private master endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + + it "has autoupgrade enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoUpgrade" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/private_zonal_with_networking/controls/network.rb b/test/integration/private_zonal_with_networking/controls/network.rb new file mode 100644 index 0000000000..a17ce74663 --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/network.rb @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +network_name = attribute('network_name') +subnet_name = attribute('subnet_name') +control "network" do + title "gcp network configuration" + describe google_compute_network( + project: project_id, + name: network_name + ) do + it { should exist } + its ('subnetworks.count') { should eq 1 } + its ('subnetworks.first') { should match subnet_name } + end + end diff --git a/test/integration/private_zonal_with_networking/controls/subnet.rb b/test/integration/private_zonal_with_networking/controls/subnet.rb new file mode 100644 index 0000000000..f88d46355b --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/subnet.rb @@ -0,0 +1,46 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +network_name = attribute('network_name') +subnet_name = attribute('subnet_name') +region = attribute('region') +ip_range_pods_name = attribute('ip_range_pods_name') +ip_range_services_name = attribute('ip_range_services_name') +control "subnet" do + title "gcp subnetwork configuration" + describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + let(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + it "#should have the correct secondaryIpRanges configuration for #{ip_range_pods_name}" do + expect(data["secondaryIpRanges"][0]).to include( + "rangeName" => ip_range_pods_name, + "ipCidrRange" => "192.168.0.0/18" + ) + end + it "#should have the correct secondaryIpRanges configuration for #{ip_range_services_name}" do + expect(data["secondaryIpRanges"][1]).to include( + "rangeName" => ip_range_services_name, + "ipCidrRange" => "192.168.64.0/18" + ) + end + end + end diff --git a/test/integration/private_zonal_with_networking/inspec.yml b/test/integration/private_zonal_with_networking/inspec.yml new file mode 100644 index 0000000000..bf2e4e86aa --- /dev/null +++ b/test/integration/private_zonal_with_networking/inspec.yml @@ -0,0 +1,36 @@ +name: simple_regional_with_networking +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: network_name + required: true + type: string + - name: subnet_name + required: true + type: string + - name: region + required: true + type: string + - name: ip_range_pods_name + required: true + type: string + - name: ip_range_services_name + required: true + type: string From c3d653690fdfeeb1a84581381742b0a57e062483 Mon Sep 17 00:00:00 2001 From: Sam Gendler Date: Wed, 6 Nov 2019 01:40:10 -0800 Subject: [PATCH 004/110] get gke module to wait on network creation correctly --- examples/private_zonal_with_networking/main.tf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index fb5d34d757..dc62661017 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -44,7 +44,7 @@ module "gcp-network" { } data "google_compute_subnetwork" "subnetwork" { - name = module.gcp-network.subnets_names[0] + name = var.subnetwork project = var.project_id region = var.region depends_on = [module.gcp-network] @@ -57,7 +57,12 @@ module "gke" { regional = false region = var.region zones = slice(var.zones, 0, 1) - network = data.google_compute_subnetwork.subnetwork.network + + // This craziness gets a plain network name from the reference link which is the + // only way to force cluster creation to wait on network creation without a + // depends_on link. Tests use terraform 0.12.6, which does not have regex or regexall + network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0] + subnetwork = data.google_compute_subnetwork.subnetwork.name ip_range_pods = var.ip_range_pods_name ip_range_services = var.ip_range_services_name From 23bfac998d8835013f83a7b312ddd86a867935d6 Mon Sep 17 00:00:00 2001 From: Sam Gendler Date: Wed, 6 Nov 2019 04:34:42 -0800 Subject: [PATCH 005/110] Get tests for private_zonal_with_network to function correctly --- .kitchen.yml | 2 +- .../private_zonal_with_networking/outputs.tf | 1 - .../test_outputs.tf | 8 +- .../variables.tf | 10 +-- .../controls/gcloud.rb | 85 +------------------ .../controls/subnet.rb | 49 ++++++----- .../private_zonal_with_networking/inspec.yml | 2 +- 7 files changed, 41 insertions(+), 116 deletions(-) diff --git a/.kitchen.yml b/.kitchen.yml index 7707328091..242d0b27c1 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -77,7 +77,7 @@ suites: backend: local controls: - gcloud - - name: subnet + - name: private_zonal_with_networking backend: local controls: - subnet diff --git a/examples/private_zonal_with_networking/outputs.tf b/examples/private_zonal_with_networking/outputs.tf index bb255b54a2..f306934801 100644 --- a/examples/private_zonal_with_networking/outputs.tf +++ b/examples/private_zonal_with_networking/outputs.tf @@ -57,4 +57,3 @@ output "subnet_secondary_ranges" { } - diff --git a/examples/private_zonal_with_networking/test_outputs.tf b/examples/private_zonal_with_networking/test_outputs.tf index a703679105..e8eeacb9ef 100644 --- a/examples/private_zonal_with_networking/test_outputs.tf +++ b/examples/private_zonal_with_networking/test_outputs.tf @@ -21,10 +21,6 @@ output "project_id" { value = var.project_id } -output "region" { - value = module.gke.region -} - output "network" { value = var.network } @@ -37,6 +33,10 @@ output "location" { value = module.gke.location } +output "region" { + value = var.region +} + output "ip_range_pods_name" { description = "The secondary IP range used for pods" value = var.ip_range_pods_name diff --git a/examples/private_zonal_with_networking/variables.tf b/examples/private_zonal_with_networking/variables.tf index ac2c68d2b6..ecaf86a558 100644 --- a/examples/private_zonal_with_networking/variables.tf +++ b/examples/private_zonal_with_networking/variables.tf @@ -25,7 +25,11 @@ variable "cluster_name" { variable "region" { description = "The region to host the cluster in" - default = "us-central1" +} + +variable "zones" { + type = list(string) + description = "The zone to host the cluster in (required if is a zonal cluster)" } variable "network" { @@ -48,7 +52,3 @@ variable "ip_range_services_name" { default = "ip-range-scv" } -variable "zones" { - default = [] -} - diff --git a/test/integration/private_zonal_with_networking/controls/gcloud.rb b/test/integration/private_zonal_with_networking/controls/gcloud.rb index e998182da2..adaf6fd646 100644 --- a/test/integration/private_zonal_with_networking/controls/gcloud.rb +++ b/test/integration/private_zonal_with_networking/controls/gcloud.rb @@ -75,53 +75,19 @@ ) ) end - end - - describe "node pool" do - let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } - - it "has autoscaling enabled" do - expect(node_pools).to include( - including( - "autoscaling" => including( - "enabled" => true, - ), - ) - ) - end - - it "has the expected minimum node count" do - expect(node_pools).to include( - including( - "autoscaling" => including( - "minNodeCount" => 1, - ), - ) - ) - end - - it "has the expected maximum node count" do - expect(node_pools).to include( - including( - "autoscaling" => including( - "maxNodeCount" => 100, - ), - ) - ) - end it "is the expected machine type" do - expect(node_pools).to include( + expect(data['nodePools']).to include( including( "config" => including( - "machineType" => "n1-standard-2", + "machineType" => "n1-standard-1", ), ) ) end it "has the expected disk size" do - expect(node_pools).to include( + expect(data['nodePools']).to include( including( "config" => including( "diskSizeGb" => 100, @@ -130,51 +96,6 @@ ) end - it "has the expected labels" do - expect(node_pools).to include( - including( - "config" => including( - "labels" => including( - "cluster_name" => cluster_name, - "node_pool" => "default-node-pool", - ), - ), - ) - ) - end - - it "has the expected network tags" do - expect(node_pools).to include( - including( - "config" => including( - "tags" => match_array([ - "gke-#{cluster_name}", - "gke-#{cluster_name}-default-node-pool", - ]), - ), - ) - ) - end - - it "has autorepair enabled" do - expect(node_pools).to include( - including( - "management" => including( - "autoRepair" => true, - ), - ) - ) - end - - it "has autoupgrade enabled" do - expect(node_pools).to include( - including( - "management" => including( - "autoUpgrade" => true, - ), - ) - ) - end end end end diff --git a/test/integration/private_zonal_with_networking/controls/subnet.rb b/test/integration/private_zonal_with_networking/controls/subnet.rb index f88d46355b..7b967b6c87 100644 --- a/test/integration/private_zonal_with_networking/controls/subnet.rb +++ b/test/integration/private_zonal_with_networking/controls/subnet.rb @@ -18,29 +18,34 @@ region = attribute('region') ip_range_pods_name = attribute('ip_range_pods_name') ip_range_services_name = attribute('ip_range_services_name') + control "subnet" do - title "gcp subnetwork configuration" - describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - let(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - it "#should have the correct secondaryIpRanges configuration for #{ip_range_pods_name}" do - expect(data["secondaryIpRanges"][0]).to include( - "rangeName" => ip_range_pods_name, - "ipCidrRange" => "192.168.0.0/18" - ) - end - it "#should have the correct secondaryIpRanges configuration for #{ip_range_services_name}" do - expect(data["secondaryIpRanges"][1]).to include( - "rangeName" => ip_range_services_name, - "ipCidrRange" => "192.168.64.0/18" - ) + title "gcp subnetwork configuration" + describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} end end + + it "#should have the correct secondaryIpRanges configuration for #{ip_range_pods_name}" do + expect(data["secondaryIpRanges"][0]).to include( + "rangeName" => ip_range_pods_name, + "ipCidrRange" => "192.168.0.0/18" + ) + end + + it "#should have the correct secondaryIpRanges configuration for #{ip_range_services_name}" do + expect(data["secondaryIpRanges"][1]).to include( + "rangeName" => ip_range_services_name, + "ipCidrRange" => "192.168.64.0/18" + ) + end + end +end diff --git a/test/integration/private_zonal_with_networking/inspec.yml b/test/integration/private_zonal_with_networking/inspec.yml index bf2e4e86aa..87f447173f 100644 --- a/test/integration/private_zonal_with_networking/inspec.yml +++ b/test/integration/private_zonal_with_networking/inspec.yml @@ -1,4 +1,4 @@ -name: simple_regional_with_networking +name: private_zonal_with_networking depends: - name: inspec-gcp git: https://github.com/inspec/inspec-gcp.git From 1e1ed912d8cc501ccdcb10f9c32233adc66144a3 Mon Sep 17 00:00:00 2001 From: Sam Gendler Date: Wed, 6 Nov 2019 05:09:36 -0800 Subject: [PATCH 006/110] lint fix --- .../private_zonal_with_networking/main.tf | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index dc62661017..82db39a591 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -22,9 +22,9 @@ module "gcp-network" { subnets = [ { - subnet_name = var.subnetwork - subnet_ip = "10.0.0.0/17" - subnet_region = var.region + subnet_name = var.subnetwork + subnet_ip = "10.0.0.0/17" + subnet_region = var.region subnet_private_access = "true" }, ] @@ -44,24 +44,24 @@ module "gcp-network" { } data "google_compute_subnetwork" "subnetwork" { - name = var.subnetwork - project = var.project_id - region = var.region + name = var.subnetwork + project = var.project_id + region = var.region depends_on = [module.gcp-network] } module "gke" { - source = "../../modules/beta-private-cluster/" - project_id = var.project_id - name = var.cluster_name - regional = false - region = var.region - zones = slice(var.zones, 0, 1) + source = "../../modules/beta-private-cluster/" + project_id = var.project_id + name = var.cluster_name + regional = false + region = var.region + zones = slice(var.zones, 0, 1) // This craziness gets a plain network name from the reference link which is the // only way to force cluster creation to wait on network creation without a // depends_on link. Tests use terraform 0.12.6, which does not have regex or regexall - network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0] + network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0] subnetwork = data.google_compute_subnetwork.subnetwork.name ip_range_pods = var.ip_range_pods_name From b3a6d3e2d4bbba0d978e85591299e279cc7e9cad Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:01:16 -0600 Subject: [PATCH 007/110] add shielded vms, use new CI, address comments --- examples/safer_cluster/main.tf | 34 ++++++++++++------- examples/safer_cluster/variables.tf | 8 +++++ modules/safer-cluster/main.tf | 18 +++------- modules/safer-cluster/variables.tf | 7 ++++ test/fixtures/safer_cluster/example.tf | 15 ++++---- .../safer_cluster/controls/gcloud.rb | 12 +++++++ 6 files changed, 62 insertions(+), 32 deletions(-) diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 7b16e90abc..102d539502 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" + version = "~> 2.18.0" region = var.region } @@ -30,17 +30,27 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/safer-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - master_ipv4_cidr_block = "172.16.0.0/28" - + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + compute_engine_service_account = var.compute_engine_service_account + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] istio = var.istio cloudrun = var.cloudrun } diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 733103bc31..deb08f59e9 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -53,3 +53,11 @@ variable "cloudrun" { default = true } +variable "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation to use for the hosted master network" + default = "172.16.0.0/28" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 1233afa65c..9069e22559 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -45,9 +45,6 @@ module "gke" { horizontal_pod_autoscaling = var.horizontal_pod_autoscaling http_load_balancing = var.http_load_balancing - // Disable the dashboard. It creates risk by running as a very sensitive user. - kubernetes_dashboard = false - // We suggest the use coarse network policies to enforce restrictions in the // communication between pods. // @@ -55,8 +52,7 @@ module "gke" { // NetworkPolicies need to be configured in every namespace. The network // policies should be under the control of a cental cluster management team, // rather than individual teams. - network_policy = true - network_policy_provider = "CALICO" + network_policy = true maintenance_start_time = var.maintenance_start_time @@ -66,8 +62,6 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - disable_legacy_metadata_endpoints = true - node_pools = var.node_pools node_pools_labels = var.node_pools_labels @@ -78,7 +72,7 @@ module "gke" { node_pools_tags = var.node_pools_tags // TODO(mmontan): we generally considered applying - // just the cloud-platofrm scope and use Cloud IAM + // just the cloud-platoform scope and use Cloud IAM // If we have Workload Identity, are there advantages // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes @@ -86,9 +80,6 @@ module "gke" { stub_domains = var.stub_domains upstream_nameservers = var.upstream_nameservers - // We should use IP Alias. - configure_ip_masq = false - logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -101,7 +92,7 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + create_service_account = var.compute_engine_service_account == "" ? false : true service_account = var.compute_engine_service_account // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, @@ -148,7 +139,6 @@ module "gke" { }] resource_usage_export_dataset_id = var.resource_usage_export_dataset_id - node_metadata = "SECURE" // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). // Sandbox can also provide increased protection in other cases, at some performance cost. @@ -163,4 +153,6 @@ module "gke" { identity_namespace = "${var.project_id}.svc.id.goog" authenticator_security_group = var.authenticator_security_group + + enable_shielded_nodes = var.enable_shielded_nodes } diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 3a3ffefca2..71b4ba88ef 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -293,3 +293,10 @@ variable "compute_engine_service_account" { description = "Use the given service account for nodes rather than creating a new dedicated service account." default = "" } + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster." + default = true +} + diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index 87285c7416..c9d659ba67 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,11 +17,12 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" - region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index e9a78d7f5e..920664593b 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -174,6 +174,18 @@ ) ) end + + it "has shielded nodes" do + expect(node_pools).to include( + including( + "config" => including( + "shieldedInstanceConfig" => including( + "enableIntegrityMonitoring" => true, + ) + ), + ) + ) + end end end end From 31f833b078863ced726d74ffaa53a875dd9b47a1 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:16:47 -0600 Subject: [PATCH 008/110] adds docs from PR281, add desc --- examples/safer_cluster/README.md | 16 +- examples/safer_cluster/outputs.tf | 13 +- modules/safer-cluster/README.md | 275 +++++++++++++++++- .../safer_cluster/controls/gcloud.rb | 4 +- 4 files changed, 287 insertions(+), 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 7cd54188f4..93a7d4a1a7 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -1,9 +1,8 @@ # Safer GKE Cluster -This example illustrates how to instantiate the opinionanted Safer Cluster module. - -[^]: (autogen_docs_start) +This example illustrates how to instantiate the opinionated Safer Cluster module. + ## Inputs | Name | Description | Type | Default | Required | @@ -11,10 +10,10 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | @@ -24,13 +23,12 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | Name | Description | |------|-------------| -| ca\_certificate | | -| client\_token | | +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| credentials\_path | | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | | +| kubernetes\_endpoint | The cluster endpoint | | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | @@ -40,7 +38,7 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 0d972dcd88..2ff9f4734b 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -15,17 +15,20 @@ */ output "kubernetes_endpoint" { - sensitive = true - value = module.gke.endpoint + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint } output "client_token" { - sensitive = true - value = base64encode(data.google_client_config.default.access_token) + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) } output "ca_certificate" { - value = module.gke.ca_certificate + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate } output "service_account" { diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 6f0206d826..ad773982be 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,12 +1,277 @@ -# Safer Beta Cluster +# Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -The module defines a safer configuration for a GKE cluster. +[TOC] -TODO(mmontan): add documentation for the module. +This module defines an opinionated setup of GKE +cluster. We outline project configurations, cluster settings, and basic K8s +objects that permit a safer-than-default configuration. -[^]: (autogen_docs_start) +## Module Usage -[^]: (autogen_docs_end) +The module fixes a set of parameters to values suggested in the +[GKE harderning guide](https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster), +the CIS framework, and other best practices. + +The motivation for each setting, and its relation to harderning guides or other recommendations +is outline in `main.tf` as comments over individual settings. When security-relevant settings +are available for configuration, recommendations on their settings are documented in the `variables.tf` file. + +## Project Setup and Cloud IAM policy for GKE + +### Applications and Clusters + +- Different applications that access data with different sensitivity and do + not need to communicate with each other (e.g., dev and prod instances of the + same application) should be placed in different clusters. + + - This approach will limit the blast radius of errors. An security problem + in dev shouldn't impact production data. + +- If applications need to communicate (e.g., a frontend system calling + a backend), we suggest placing the two applications in the same cluster, in + different namespaces. + + - Placing them in the same cluster will provide fast network + communication, and the different namespaces will be configured to + provide some administrative isolation. Istio will be used to encrypt and + control communication between applications. + +- We suggest to store user or business data persistently in managed storage + services that are inventories and controlled by centralized teams. + (e.g., GCP storage services within a GCP organization). + + - Storing user or business data securely requires satisfying a large set of + requirements, such as data inventory, which might be harder to satisfy at + scale when data is stored opaquely within a cluster. Services like Cloud + Asset Inventory provide centralized teams ability to enumerate data stores. + +### Project Setup + +We suggest a GKE setup composed of multiple projects to separate responsabilities +between cluster operators, which need to administer the cluster; and product +developers, which mostly just want to deploy and debug applications. + +- *Cluster Projects (`project_id`):* GKE clusters storing sensitive data should have their + own projects, so that they can be administered independently (e.g., dev cluster; + production clusters; staging clusters should go in different projects.) + +- *A shared GCR project (`registry_project_id`):* all clusters can share the same GCR project. + + - Easier to share images between environments. The same image could be + progressively rolled-out in dev, staging, and then production. + - Easier to manage service account permissions: GCR requires authorizing + service accounts to access certain buckets, which are created only after + images are published. When the only service run by the project is GCR, + we can safely give project-wide read permissions to all buckets. + +- (optional) *Data Projects:* When the same cluster is shared by different + applications managed by different teams, we suggest separating the data for + each application by placing storage resources for each team in different + projects (e.g., a Spanner instance for application A in one project, GCS + bucket for application B in a different project). + + - This permits to control administrative access to the data more tightly, + as Cloud IAM policies for accessing the data can be managed by each + application team, rather than the team managing the cluster + infrastructure. + +Exception to such a setup: + +- When not using Shared VPCs, resources that require direct network connectivity + (e.g., a Cloud SQL instance), need to be placed in the same VPC (hence, project) + as the clusters from which connections are made. + +### Google Service Accounts + +We use GKE Workload Identity (BETA) to associate a GCP identity to each workload, +and limit the permissions associated with the cluster nodes. + +The Safer Cluster setup relies on several service accounts: + +- The module generates a service account to run nodes. Such a service account + has only permissions of sending logging data, metrics, and downloading containers + from the given GCR project. The following settings in the module will create + a service account with the above properties: + +``` +create_service_account = true +registry_project_id = +grant_registry_access = true +``` + +- A service account *for each application* running on the cluster (e.g., + `myproduct-sa@myproduct-prod.iam.gserviceaccount.com`). These service + accounts should be associated to the permissions required for running the + application, such as access to databases. + +``` +- email: myproduct + displayName: Google Service Account for containers running in the myproduct k8s namespace + policy: + # GKE workload identity authorization. This authorizes the Kubernetes Service Account + # myproduct/default from this project's identity namespace to impersonate the service account. + # https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + bindings: + - members: + - serviceAccount:product-prod.svc.id.goog[myproduct/default] + role: roles/iam.workloadIdentityUser +``` + +We suggest running different applications in different namespaces within the cluster. Each namespace +should be assigned to its own GCP service account to better define the Cloud IAM permissions required +for the application. + +If you are using more than 2 projects in your setup, you can consider creting +the service account in a different project to keep application and +infrastructure separate. For example, service accounts could be created in each team's project, +while the cluster runs in a centrally controlled project. + +
+*Why?* + +Separating the permissions associated with the infrastructure GKE nodes and the +application provides a simpler way to scale up the cluster: multiple applications +could be run in the same cluster, and each of them can run with tailored permissions +that limit the impact of compromises. + +Such a separation of identities is enabled by a GKE feature called Workload +Identity. The feature provides additional advantages such as a better protection +of the node's metadata server against attackers. + +
+ +### Cloud IAM Permissions for the GKE Cluster + +We suggest to mainly rely on Kubernetes RBAC to manage access control, and use +Cloud IAM to give users only the ability of configuring `kubectl` credentials. + +Engineers operating applications on the cluster should only be assigned the +Cloud IAM permission `roles/container.clusterViewer`. This role allows them to +obtain credentials for the cluster, but provides no further access to the +cluster objects. All cluster objects are protected by RBAC configurations, +defined below. + +
+*Why?* + +Both Cloud IAM and RBAC can be used to control access to GKE clusters. Those two +systems are combined as a "OR": an action is authorized if the necessary +permissions are provided by either RBAC _OR_ Cloud IAM + +However, Cloud IAM permissions are defined for a project: user get assigned the +same permissions over all clusters and all namespaces within each cluster. Such +a setup makes it hard to separate responsibilities between teams in charge of +managing clusters, and teams in charge of products. + +By relying on RBAC instead of Cloud IAM, we have a finer-grained control of the +permissions provided to engineers, and permits to restrict permissions to only +certain namespaces. + +
+ +You can add the following binding to the `myproduct-prod` project. + +``` +- members: + role: roles/container.clusterViewer` + - group: + - group: +``` + +The permissions won't allow engineers to SSH into nodes as part of the regular +development workflow. Such permissions should be granted only to the cluster +team, and used only in case of emergency. + +While RBAC permissions should be sufficient for most cases, we also suggest to +create an emergency superuser role that can be used, given a proper +justification, for resolving cases where regular permissions are insufficient. +For simplicity, we suggest using `roles/container.admin` and +`roles/compute.admin`, until more narrow roles can be defined given your usage. + +``` +- members: + role: roles/container.admin + - group: +- members: + role: roles/compute.admin + - group: +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | string | `""` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index 920664593b..c2cdd7bca0 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -1,10 +1,10 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From 2c1aa19fb53ae134b6be2ebb0e3a5b4200d4911f Mon Sep 17 00:00:00 2001 From: Mirko Montanari Date: Sun, 22 Sep 2019 12:19:43 -0700 Subject: [PATCH 009/110] Initial definition of a Safer Cluster module. The PR adds a new "safer" GKE module. The module includes hardening suggestions from multiple sources. --- .kitchen.yml | 7 + examples/safer_cluster/README.md | 49 +++ examples/safer_cluster/main.tf | 50 +++ examples/safer_cluster/outputs.tf | 35 +++ examples/safer_cluster/test_outputs.tf | 1 + examples/safer_cluster/variables.tf | 55 ++++ examples/safer_cluster/versions.tf | 19 ++ modules/safer-cluster/README.md | 15 + modules/safer-cluster/main.tf | 166 ++++++++++ modules/safer-cluster/outputs.tf | 123 ++++++++ modules/safer-cluster/variables.tf | 295 ++++++++++++++++++ modules/safer-cluster/versions.tf | 19 ++ test/ci/safer-cluster.yml | 18 ++ test/fixtures/safer_cluster/example.tf | 27 ++ test/fixtures/safer_cluster/network.tf | 48 +++ test/fixtures/safer_cluster/outputs.tf | 1 + test/fixtures/safer_cluster/variables.tf | 1 + .../safer_cluster/controls/gcloud.rb | 179 +++++++++++ test/integration/safer_cluster/inspec.yml | 17 + 19 files changed, 1125 insertions(+) create mode 100644 examples/safer_cluster/README.md create mode 100644 examples/safer_cluster/main.tf create mode 100644 examples/safer_cluster/outputs.tf create mode 120000 examples/safer_cluster/test_outputs.tf create mode 100644 examples/safer_cluster/variables.tf create mode 100644 examples/safer_cluster/versions.tf create mode 100644 modules/safer-cluster/README.md create mode 100644 modules/safer-cluster/main.tf create mode 100644 modules/safer-cluster/outputs.tf create mode 100644 modules/safer-cluster/variables.tf create mode 100644 modules/safer-cluster/versions.tf create mode 100644 test/ci/safer-cluster.yml create mode 100644 test/fixtures/safer_cluster/example.tf create mode 100644 test/fixtures/safer_cluster/network.tf create mode 120000 test/fixtures/safer_cluster/outputs.tf create mode 120000 test/fixtures/safer_cluster/variables.tf create mode 100644 test/integration/safer_cluster/controls/gcloud.rb create mode 100644 test/integration/safer_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 39faa2e1e3..3d7be4b20b 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -45,6 +45,13 @@ suites: systems: - name: shared_vpc backend: local + - name: "safer_cluster" + driver: + root_module_directory: test/fixtures/safer_cluster + verifier: + systems: + - name: safer_cluster + backend: local - name: "simple_regional" driver: root_module_directory: test/fixtures/simple_regional diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md new file mode 100644 index 0000000000..7cd54188f4 --- /dev/null +++ b/examples/safer_cluster/README.md @@ -0,0 +1,49 @@ +# Safer GKE Cluster + +This example illustrates how to instantiate the opinionanted Safer Cluster module. + +[^]: (autogen_docs_start) + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| network | The VPC network to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | | +| client\_token | | +| cluster\_name | Cluster name | +| credentials\_path | | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | | +| location | | +| master\_kubernetes\_version | The master Kubernetes version | +| network | | +| project\_id | | +| region | | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnetwork | | +| zones | List of zones in which the cluster resides | + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf new file mode 100644 index 0000000000..7b16e90abc --- /dev/null +++ b/examples/safer_cluster/main.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "safer-cluster" +} + +provider "google-beta" { + version = "~> 2.12.0" + region = var.region +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region +} + +module "gke" { + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + master_ipv4_cidr_block = "172.16.0.0/28" + + istio = var.istio + cloudrun = var.cloudrun +} + +data "google_client_config" "default" { +} + diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf new file mode 100644 index 0000000000..0d972dcd88 --- /dev/null +++ b/examples/safer_cluster/outputs.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} + diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf new file mode 120000 index 0000000000..17b34213ba --- /dev/null +++ b/examples/safer_cluster/test_outputs.tf @@ -0,0 +1 @@ +../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf new file mode 100644 index 0000000000..733103bc31 --- /dev/null +++ b/examples/safer_cluster/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" +} + +variable "istio" { + description = "Boolean to enable / disable Istio" + default = true +} + +variable "cloudrun" { + description = "Boolean to enable / disable CloudRun" + default = true +} + diff --git a/examples/safer_cluster/versions.tf b/examples/safer_cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/examples/safer_cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md new file mode 100644 index 0000000000..6f0206d826 --- /dev/null +++ b/modules/safer-cluster/README.md @@ -0,0 +1,15 @@ +# Safer Beta Cluster + +The module defines a safer configuration for a GKE cluster. + +TODO(mmontan): add documentation for the module. + +[^]: (autogen_docs_start) + +[^]: (autogen_docs_end) + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf new file mode 100644 index 0000000000..1233afa65c --- /dev/null +++ b/modules/safer-cluster/main.tf @@ -0,0 +1,166 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// The safer-cluster module is based on a private cluster, with a several +// settings set to recommended values by default. +module "gke" { + source = "../beta-private-cluster/" + project_id = var.project_id + name = var.name + regional = var.regional + region = var.region + network = var.network + network_project_id = var.network_project_id + + // We need to enforce a minimum Kubernetes Version to ensure + // that the necessary security features are enabled. + kubernetes_version = "latest" + + // Nodes are created with a default version. The nodepool enables + // auto_upgrade so that the node versions can be kept up to date with + // the master upgrades. + // + // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades + node_version = "" + + master_authorized_networks_config = var.master_authorized_networks_config + + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + + horizontal_pod_autoscaling = var.horizontal_pod_autoscaling + http_load_balancing = var.http_load_balancing + + // Disable the dashboard. It creates risk by running as a very sensitive user. + kubernetes_dashboard = false + + // We suggest the use coarse network policies to enforce restrictions in the + // communication between pods. + // + // NOTE: Enabling network policy is not sufficient to enforce restrictions. + // NetworkPolicies need to be configured in every namespace. The network + // policies should be under the control of a cental cluster management team, + // rather than individual teams. + network_policy = true + network_policy_provider = "CALICO" + + maintenance_start_time = var.maintenance_start_time + + initial_node_count = var.initial_node_count + + // We suggest removing the default node pull, as it cannot be modified without + // destroying the cluster. + remove_default_node_pool = true + + disable_legacy_metadata_endpoints = true + + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels + + // TODO(mmontan): check whether we need to restrict these + // settings. + node_pools_metadata = var.node_pools_metadata + node_pools_taints = var.node_pools_taints + node_pools_tags = var.node_pools_tags + + // TODO(mmontan): we generally considered applying + // just the cloud-platofrm scope and use Cloud IAM + // If we have Workload Identity, are there advantages + // in restricting scopes even more? + node_pools_oauth_scopes = var.node_pools_oauth_scopes + + stub_domains = var.stub_domains + upstream_nameservers = var.upstream_nameservers + + // We should use IP Alias. + configure_ip_masq = false + + logging_service = var.logging_service + monitoring_service = var.monitoring_service + + // We never use the default service account for the cluster. The default + // project/editor permissions can create problems if nodes were to be ever + // compromised. + + // We either: + // - Create a dedicated service account with minimal permissions to run nodes. + // All applications shuold run with an identity defined via Workload Identity anyway. + // - Use a service account passed as a parameter to the module, in case the user + // wants to maintain control of their service accounts. + create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + service_account = var.compute_engine_service_account + + // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, + // so that we can give GCS permissions to the service account on a project + // that hosts only container-images and not data. + grant_registry_access = true + + // Basic Auth disabled + basic_auth_username = "" + basic_auth_password = "" + + issue_client_certificate = false + + cluster_ipv4_cidr = var.cluster_ipv4_cidr + + cluster_resource_labels = var.cluster_resource_labels + + // We enable private endpoints to limit exposure. + enable_private_endpoint = true + deploy_using_private_endpoint = true + + // Private nodes better control public exposure, and reduce + // the ability of nodes to reach to the Internet without + // additional configurations. + enable_private_nodes = true + + master_ipv4_cidr_block = var.master_ipv4_cidr_block + + // Istio is recommended for pod-to-pod communications. + istio = var.istio + cloudrun = var.cloudrun + + default_max_pods_per_node = var.default_max_pods_per_node + + database_encryption = var.database_encryption + + // We suggest to define policies about which images can run on a cluster. + enable_binary_authorization = true + + // Define PodSecurityPolicies for differnet applications. + // TODO(mmontan): link to a couple of policies. + pod_security_policy_config = [{ + "enabled" = true + }] + + resource_usage_export_dataset_id = var.resource_usage_export_dataset_id + node_metadata = "SECURE" + + // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). + // Sandbox can also provide increased protection in other cases, at some performance cost. + sandbox_enabled = var.sandbox_enabled + + // TODO(mmontan): investigate whether this should be a recommended setting + enable_intranode_visibility = var.enable_intranode_visibility + + enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling + + // We enable identity namespace by default. + identity_namespace = "${var.project_id}.svc.id.goog" + + authenticator_security_group = var.authenticator_security_group +} diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf new file mode 100644 index 0000000000..bb4fb79667 --- /dev/null +++ b/modules/safer-cluster/outputs.tf @@ -0,0 +1,123 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +output "name" { + description = "Cluster name" + value = module.gke.name +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = module.gke.type +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = module.gke.location +} + +output "region" { + description = "Cluster region" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = module.gke.endpoint + depends_on = [ + /* Nominally, the endpoint is populated as soon as it is known to Terraform. + * However, the cluster may not be in a usable state yet. Therefore any + * resources dependent on the cluster being up will fail to deploy. With + * this explicit dependency, dependent resources can wait for the cluster + * to be up. + */ + module.gke + ] +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = module.gke.min_master_version +} + +output "logging_service" { + description = "Logging service used" + value = module.gke.logging_service +} + +output "monitoring_service" { + description = "Monitoring service used" + value = module.gke.monitoring_service +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = module.gke.master_authorized_networks_config +} + +output "master_version" { + description = "Current master kubernetes version" + value = module.gke.master_version +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = module.gke.network_policy_enabled +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = module.gke.http_load_balancing_enabled +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = module.gke.horizontal_pod_autoscaling_enabled +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = module.gke.kubernetes_dashboard_enabled +} + +output "node_pools_names" { + description = "List of node pools names" + value = module.gke.node_pools_names +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = module.gke.node_pools_versions +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.gke.service_account +} diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf new file mode 100644 index 0000000000..3a3ffefca2 --- /dev/null +++ b/modules/safer-cluster/variables.tf @@ -0,0 +1,295 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +variable "project_id" { + type = string + description = "The project ID to host the cluster in (required)" +} + +variable "name" { + type = string + description = "The name of the cluster (required)" +} + +variable "description" { + type = string + description = "The description of the cluster" + default = "" +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} + +variable "region" { + type = string + description = "The region to host the cluster in (required)" +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "network" { + type = string + description = "The VPC network to host the cluster in (required)" +} + +variable "network_project_id" { + type = string + description = "The project ID of the shared VPC's host (for shared vpc support)" + default = "" +} + +variable "subnetwork" { + type = string + description = "The subnetwork to host the cluster in (required)" +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. " + default = "latest" +} + +variable "node_version" { + type = string + description = "The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation." + default = "" +} + +variable "master_authorized_networks_config" { + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) + description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." + default = [] +} + +variable "horizontal_pod_autoscaling" { + type = bool + description = "Enable horizontal pod autoscaling addon" + default = true +} + +variable "http_load_balancing" { + type = bool + description = "Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed." + default = true +} + +variable "maintenance_start_time" { + type = string + description = "Time window specified for daily maintenance operations in RFC3339 format" + default = "05:00" +} + +variable "ip_range_pods" { + type = string + description = "The _name_ of the secondary subnet ip range to use for pods" +} + +variable "ip_range_services" { + type = string + description = "The _name_ of the secondary subnet range to use for services" +} + +variable "initial_node_count" { + type = number + description = "The number of nodes to create in this cluster's default node pool." + default = 0 +} + +variable "node_pools" { + type = list(map(string)) + description = "List of maps containing node pools" + + default = [ + { + name = "default-node-pool" + }, + ] +} + +variable "node_pools_labels" { + type = map(map(string)) + description = "Map of maps containing node labels by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_metadata" { + type = map(map(string)) + description = "Map of maps containing node metadata by node-pool name" + + default = { + all = {} + default-node-pool = {} + } +} + +variable "node_pools_taints" { + type = map(list(object({ key = string, value = string, effect = string }))) + description = "Map of lists containing node taints by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_tags" { + type = map(list(string)) + description = "Map of lists containing node network tags by node-pool name" + + default = { + all = [] + default-node-pool = [] + } +} + +variable "node_pools_oauth_scopes" { + type = map(list(string)) + description = "Map of lists containing node oauth scopes by node-pool name" + + default = { + all = ["https://www.googleapis.com/auth/cloud-platform"] + default-node-pool = [] + } +} + +variable "stub_domains" { + type = map(list(string)) + description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" + default = {} +} + +variable "upstream_nameservers" { + type = "list" + description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" + default = [] +} + +variable "logging_service" { + type = string + description = "The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none" + default = "logging.googleapis.com" +} + +variable "monitoring_service" { + type = string + description = "The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none" + default = "monitoring.googleapis.com" +} + +variable "grant_registry_access" { + type = bool + description = "Grants created cluster-specific service account storage.objectViewer role." + default = false +} + +// TODO(mmontan): allow specifying which project to use +// for reading images. + +variable "service_account" { + type = string + description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." + default = "" +} + +variable "cluster_ipv4_cidr" { + default = "" + description = "The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR." +} + +variable "cluster_resource_labels" { + type = map(string) + description = "The GCE resource labels (a map of key/value pairs) to be applied to the cluster" + default = {} +} + +variable "master_ipv4_cidr_block" { + type = string + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" +} + +variable "istio" { + description = "(Beta) Enable Istio addon" + default = false +} + +variable "default_max_pods_per_node" { + description = "The maximum number of pods to schedule per node" + default = 110 +} + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "cloudrun" { + description = "(Beta) Enable CloudRun addon" + default = false +} + +variable "resource_usage_export_dataset_id" { + type = string + description = "The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic." + default = "" +} + +variable "sandbox_enabled" { + type = bool + description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it)." + default = false +} + +variable "enable_intranode_visibility" { + type = bool + description = "Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network" + default = false +} + +variable "enable_vertical_pod_autoscaling" { + type = bool + description = "Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it" + default = false +} + +variable "authenticator_security_group" { + type = string + description = "The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com" + default = null +} + +variable "compute_engine_service_account" { + type = string + description = "Use the given service account for nodes rather than creating a new dedicated service account." + default = "" +} diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf new file mode 100644 index 0000000000..832ec1df39 --- /dev/null +++ b/modules/safer-cluster/versions.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.12" +} diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml new file mode 100644 index 0000000000..ef3c896b29 --- /dev/null +++ b/test/ci/safer-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "safer-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf new file mode 100644 index 0000000000..87285c7416 --- /dev/null +++ b/test/fixtures/safer_cluster/example.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/safer_cluster" + + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf new file mode 100644 index 0000000000..e1292eae3b --- /dev/null +++ b/test/fixtures/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_id +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 120000 index 0000000000..726bdc722f --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1 @@ +../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/variables.tf b/test/fixtures/safer_cluster/variables.tf new file mode 120000 index 0000000000..c113c00a3d --- /dev/null +++ b/test/fixtures/safer_cluster/variables.tf @@ -0,0 +1 @@ +../shared/variables.tf \ No newline at end of file diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..e9a78d7f5e --- /dev/null +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -0,0 +1,179 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is regional" do + expect(data['location']).to match(/^.*[1-9]$/) + end + + it "uses the private endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => {}, + }) + end + end + + it "has network policy enabled" do + expect(data['networkPolicy']).to eq({ + "enabled" => true, + "provider" => "CALICO", + }) + end + + it "has binary authorization" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "no legacy ABAC" do + expect(data['legacyAbac']).to eq({}) + end + + describe "default node pool" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-node-pool", + ) + ) + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/safer_cluster/inspec.yml b/test/integration/safer_cluster/inspec.yml new file mode 100644 index 0000000000..b7174cb88e --- /dev/null +++ b/test/integration/safer_cluster/inspec.yml @@ -0,0 +1,17 @@ +name: safer_cluster +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string From b41137159403d177ebe6427feb98be44950f65ac Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:01:16 -0600 Subject: [PATCH 010/110] add shielded vms, use new CI, address comments --- examples/safer_cluster/main.tf | 34 ++++++++++++------- examples/safer_cluster/variables.tf | 8 +++++ modules/safer-cluster/main.tf | 18 +++------- modules/safer-cluster/variables.tf | 7 ++++ test/fixtures/safer_cluster/example.tf | 15 ++++---- .../safer_cluster/controls/gcloud.rb | 12 +++++++ 6 files changed, 62 insertions(+), 32 deletions(-) diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 7b16e90abc..102d539502 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -19,7 +19,7 @@ locals { } provider "google-beta" { - version = "~> 2.12.0" + version = "~> 2.18.0" region = var.region } @@ -30,17 +30,27 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "../../modules/safer-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - master_ipv4_cidr_block = "172.16.0.0/28" - + source = "../../modules/safer-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + compute_engine_service_account = var.compute_engine_service_account + master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] istio = var.istio cloudrun = var.cloudrun } diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 733103bc31..deb08f59e9 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -53,3 +53,11 @@ variable "cloudrun" { default = true } +variable "master_ipv4_cidr_block" { + description = "The IP range in CIDR notation to use for the hosted master network" + default = "172.16.0.0/28" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 1233afa65c..9069e22559 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -45,9 +45,6 @@ module "gke" { horizontal_pod_autoscaling = var.horizontal_pod_autoscaling http_load_balancing = var.http_load_balancing - // Disable the dashboard. It creates risk by running as a very sensitive user. - kubernetes_dashboard = false - // We suggest the use coarse network policies to enforce restrictions in the // communication between pods. // @@ -55,8 +52,7 @@ module "gke" { // NetworkPolicies need to be configured in every namespace. The network // policies should be under the control of a cental cluster management team, // rather than individual teams. - network_policy = true - network_policy_provider = "CALICO" + network_policy = true maintenance_start_time = var.maintenance_start_time @@ -66,8 +62,6 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - disable_legacy_metadata_endpoints = true - node_pools = var.node_pools node_pools_labels = var.node_pools_labels @@ -78,7 +72,7 @@ module "gke" { node_pools_tags = var.node_pools_tags // TODO(mmontan): we generally considered applying - // just the cloud-platofrm scope and use Cloud IAM + // just the cloud-platoform scope and use Cloud IAM // If we have Workload Identity, are there advantages // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes @@ -86,9 +80,6 @@ module "gke" { stub_domains = var.stub_domains upstream_nameservers = var.upstream_nameservers - // We should use IP Alias. - configure_ip_masq = false - logging_service = var.logging_service monitoring_service = var.monitoring_service @@ -101,7 +92,7 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = length(var.compute_engine_service_account) > 0 ? false : true + create_service_account = var.compute_engine_service_account == "" ? false : true service_account = var.compute_engine_service_account // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, @@ -148,7 +139,6 @@ module "gke" { }] resource_usage_export_dataset_id = var.resource_usage_export_dataset_id - node_metadata = "SECURE" // Sandbox is needed if the cluster is going to run any untrusted workload (e.g., user submitted code). // Sandbox can also provide increased protection in other cases, at some performance cost. @@ -163,4 +153,6 @@ module "gke" { identity_namespace = "${var.project_id}.svc.id.goog" authenticator_security_group = var.authenticator_security_group + + enable_shielded_nodes = var.enable_shielded_nodes } diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 3a3ffefca2..71b4ba88ef 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -293,3 +293,10 @@ variable "compute_engine_service_account" { description = "Use the given service account for nodes rather than creating a new dedicated service account." default = "" } + +variable "enable_shielded_nodes" { + type = bool + description = "Enable Shielded Nodes features on all nodes in this cluster." + default = true +} + diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index 87285c7416..c9d659ba67 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,11 +17,12 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" - region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index e9a78d7f5e..920664593b 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -174,6 +174,18 @@ ) ) end + + it "has shielded nodes" do + expect(node_pools).to include( + including( + "config" => including( + "shieldedInstanceConfig" => including( + "enableIntegrityMonitoring" => true, + ) + ), + ) + ) + end end end end From 8539316fe1708ed0555c43f64cbd0f59f26f6ba8 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 11 Nov 2019 08:16:47 -0600 Subject: [PATCH 011/110] adds docs from PR281, add desc --- examples/safer_cluster/README.md | 16 +- examples/safer_cluster/outputs.tf | 13 +- modules/safer-cluster/README.md | 275 +++++++++++++++++- .../safer_cluster/controls/gcloud.rb | 4 +- 4 files changed, 287 insertions(+), 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 7cd54188f4..93a7d4a1a7 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -1,9 +1,8 @@ # Safer GKE Cluster -This example illustrates how to instantiate the opinionanted Safer Cluster module. - -[^]: (autogen_docs_start) +This example illustrates how to instantiate the opinionated Safer Cluster module. + ## Inputs | Name | Description | Type | Default | Required | @@ -11,10 +10,10 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| credentials\_path | The path to the GCP credentials JSON file | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | @@ -24,13 +23,12 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | Name | Description | |------|-------------| -| ca\_certificate | | -| client\_token | | +| ca\_certificate | The cluster ca certificate (base64 encoded) | +| client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| credentials\_path | | | ip\_range\_pods | The secondary IP range used for pods | | ip\_range\_services | The secondary IP range used for services | -| kubernetes\_endpoint | | +| kubernetes\_endpoint | The cluster endpoint | | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | @@ -40,7 +38,7 @@ This example illustrates how to instantiate the opinionanted Safer Cluster modul | subnetwork | | | zones | List of zones in which the cluster resides | -[^]: (autogen_docs_end) + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 0d972dcd88..2ff9f4734b 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -15,17 +15,20 @@ */ output "kubernetes_endpoint" { - sensitive = true - value = module.gke.endpoint + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint } output "client_token" { - sensitive = true - value = base64encode(data.google_client_config.default.access_token) + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) } output "ca_certificate" { - value = module.gke.ca_certificate + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate } output "service_account" { diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 6f0206d826..ad773982be 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,12 +1,277 @@ -# Safer Beta Cluster +# Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -The module defines a safer configuration for a GKE cluster. +[TOC] -TODO(mmontan): add documentation for the module. +This module defines an opinionated setup of GKE +cluster. We outline project configurations, cluster settings, and basic K8s +objects that permit a safer-than-default configuration. -[^]: (autogen_docs_start) +## Module Usage -[^]: (autogen_docs_end) +The module fixes a set of parameters to values suggested in the +[GKE harderning guide](https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster), +the CIS framework, and other best practices. + +The motivation for each setting, and its relation to harderning guides or other recommendations +is outline in `main.tf` as comments over individual settings. When security-relevant settings +are available for configuration, recommendations on their settings are documented in the `variables.tf` file. + +## Project Setup and Cloud IAM policy for GKE + +### Applications and Clusters + +- Different applications that access data with different sensitivity and do + not need to communicate with each other (e.g., dev and prod instances of the + same application) should be placed in different clusters. + + - This approach will limit the blast radius of errors. An security problem + in dev shouldn't impact production data. + +- If applications need to communicate (e.g., a frontend system calling + a backend), we suggest placing the two applications in the same cluster, in + different namespaces. + + - Placing them in the same cluster will provide fast network + communication, and the different namespaces will be configured to + provide some administrative isolation. Istio will be used to encrypt and + control communication between applications. + +- We suggest to store user or business data persistently in managed storage + services that are inventories and controlled by centralized teams. + (e.g., GCP storage services within a GCP organization). + + - Storing user or business data securely requires satisfying a large set of + requirements, such as data inventory, which might be harder to satisfy at + scale when data is stored opaquely within a cluster. Services like Cloud + Asset Inventory provide centralized teams ability to enumerate data stores. + +### Project Setup + +We suggest a GKE setup composed of multiple projects to separate responsabilities +between cluster operators, which need to administer the cluster; and product +developers, which mostly just want to deploy and debug applications. + +- *Cluster Projects (`project_id`):* GKE clusters storing sensitive data should have their + own projects, so that they can be administered independently (e.g., dev cluster; + production clusters; staging clusters should go in different projects.) + +- *A shared GCR project (`registry_project_id`):* all clusters can share the same GCR project. + + - Easier to share images between environments. The same image could be + progressively rolled-out in dev, staging, and then production. + - Easier to manage service account permissions: GCR requires authorizing + service accounts to access certain buckets, which are created only after + images are published. When the only service run by the project is GCR, + we can safely give project-wide read permissions to all buckets. + +- (optional) *Data Projects:* When the same cluster is shared by different + applications managed by different teams, we suggest separating the data for + each application by placing storage resources for each team in different + projects (e.g., a Spanner instance for application A in one project, GCS + bucket for application B in a different project). + + - This permits to control administrative access to the data more tightly, + as Cloud IAM policies for accessing the data can be managed by each + application team, rather than the team managing the cluster + infrastructure. + +Exception to such a setup: + +- When not using Shared VPCs, resources that require direct network connectivity + (e.g., a Cloud SQL instance), need to be placed in the same VPC (hence, project) + as the clusters from which connections are made. + +### Google Service Accounts + +We use GKE Workload Identity (BETA) to associate a GCP identity to each workload, +and limit the permissions associated with the cluster nodes. + +The Safer Cluster setup relies on several service accounts: + +- The module generates a service account to run nodes. Such a service account + has only permissions of sending logging data, metrics, and downloading containers + from the given GCR project. The following settings in the module will create + a service account with the above properties: + +``` +create_service_account = true +registry_project_id = +grant_registry_access = true +``` + +- A service account *for each application* running on the cluster (e.g., + `myproduct-sa@myproduct-prod.iam.gserviceaccount.com`). These service + accounts should be associated to the permissions required for running the + application, such as access to databases. + +``` +- email: myproduct + displayName: Google Service Account for containers running in the myproduct k8s namespace + policy: + # GKE workload identity authorization. This authorizes the Kubernetes Service Account + # myproduct/default from this project's identity namespace to impersonate the service account. + # https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + bindings: + - members: + - serviceAccount:product-prod.svc.id.goog[myproduct/default] + role: roles/iam.workloadIdentityUser +``` + +We suggest running different applications in different namespaces within the cluster. Each namespace +should be assigned to its own GCP service account to better define the Cloud IAM permissions required +for the application. + +If you are using more than 2 projects in your setup, you can consider creting +the service account in a different project to keep application and +infrastructure separate. For example, service accounts could be created in each team's project, +while the cluster runs in a centrally controlled project. + +
+*Why?* + +Separating the permissions associated with the infrastructure GKE nodes and the +application provides a simpler way to scale up the cluster: multiple applications +could be run in the same cluster, and each of them can run with tailored permissions +that limit the impact of compromises. + +Such a separation of identities is enabled by a GKE feature called Workload +Identity. The feature provides additional advantages such as a better protection +of the node's metadata server against attackers. + +
+ +### Cloud IAM Permissions for the GKE Cluster + +We suggest to mainly rely on Kubernetes RBAC to manage access control, and use +Cloud IAM to give users only the ability of configuring `kubectl` credentials. + +Engineers operating applications on the cluster should only be assigned the +Cloud IAM permission `roles/container.clusterViewer`. This role allows them to +obtain credentials for the cluster, but provides no further access to the +cluster objects. All cluster objects are protected by RBAC configurations, +defined below. + +
+*Why?* + +Both Cloud IAM and RBAC can be used to control access to GKE clusters. Those two +systems are combined as a "OR": an action is authorized if the necessary +permissions are provided by either RBAC _OR_ Cloud IAM + +However, Cloud IAM permissions are defined for a project: user get assigned the +same permissions over all clusters and all namespaces within each cluster. Such +a setup makes it hard to separate responsibilities between teams in charge of +managing clusters, and teams in charge of products. + +By relying on RBAC instead of Cloud IAM, we have a finer-grained control of the +permissions provided to engineers, and permits to restrict permissions to only +certain namespaces. + +
+ +You can add the following binding to the `myproduct-prod` project. + +``` +- members: + role: roles/container.clusterViewer` + - group: + - group: +``` + +The permissions won't allow engineers to SSH into nodes as part of the regular +development workflow. Such permissions should be granted only to the cluster +team, and used only in case of emergency. + +While RBAC permissions should be sufficient for most cases, we also suggest to +create an emergency superuser role that can be used, given a proper +justification, for resolving cases where regular permissions are insufficient. +For simplicity, we suggest using `roles/container.admin` and +`roles/compute.admin`, until more narrow roles can be defined given your usage. + +``` +- members: + role: roles/container.admin + - group: +- members: + role: roles/compute.admin + - group: +``` + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| compute\_engine\_service\_account | Use the given service account for nodes rather than creating a new dedicated service account. | string | `""` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster. | bool | `"true"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon. The addon allows whoever can create Ingress objects to expose an application to a public IP. Network policies or Gatekeeper policies should be used to verify that only authorized applications are exposed. | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (required) | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + + To provision this example, run the following from within this directory: - `terraform init` to get the plugins diff --git a/test/integration/safer_cluster/controls/gcloud.rb b/test/integration/safer_cluster/controls/gcloud.rb index 920664593b..c2cdd7bca0 100644 --- a/test/integration/safer_cluster/controls/gcloud.rb +++ b/test/integration/safer_cluster/controls/gcloud.rb @@ -1,10 +1,10 @@ -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, From 5a194719faa144ad0a7ee578663d336358f5073c Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Wed, 13 Nov 2019 14:03:23 -0600 Subject: [PATCH 012/110] address comments --- examples/safer_cluster/README.md | 15 ++++--- examples/safer_cluster/main.tf | 7 ++-- examples/safer_cluster/network.tf | 48 ++++++++++++++++++++++ examples/safer_cluster/outputs.tf | 10 +++++ examples/safer_cluster/variables.tf | 19 ++++++++- modules/safer-cluster/README.md | 9 ++-- modules/safer-cluster/main.tf | 17 +++----- modules/safer-cluster/variables.tf | 7 +++- test/ci/safer-cluster.yml | 18 -------- test/fixtures/safer_cluster/example.tf | 5 --- test/fixtures/safer_cluster/network.tf | 48 ---------------------- test/fixtures/safer_cluster/outputs.tf | 57 +++++++++++++++++++++++++- 12 files changed, 158 insertions(+), 102 deletions(-) create mode 100644 examples/safer_cluster/network.tf delete mode 100644 test/ci/safer-cluster.yml delete mode 100644 test/fixtures/safer_cluster/network.tf mode change 120000 => 100644 test/fixtures/safer_cluster/outputs.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 93a7d4a1a7..ba26e59314 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -10,14 +10,17 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | +| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | | istio | Boolean to enable / disable Istio | string | `"true"` | no | +| master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | +| master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | n/a | yes | +| network | The VPC network to host the cluster in | string | `"gke-network"` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | `"us-central1"` | no | +| subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | +| subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | ## Outputs @@ -32,9 +35,11 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | +| network\_name | The name of the VPC being created | | project\_id | | | region | | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnet\_names | The names of the subnet being created | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 102d539502..e865c9c2f9 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -20,7 +20,6 @@ locals { provider "google-beta" { version = "~> 2.18.0" - region = var.region } data "google_compute_subnetwork" "subnetwork" { @@ -35,8 +34,8 @@ module "gke" { name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" regional = true region = var.region - network = var.network - subnetwork = var.subnetwork + network = module.gcp-network.network_name + subnetwork = module.gcp-network.subnets_names[0] ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services compute_engine_service_account = var.compute_engine_service_account @@ -45,7 +44,7 @@ module "gke" { { cidr_blocks = [ { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + cidr_block = var.master_auth_subnetwork_cidr display_name = "VPC" }, ] diff --git a/examples/safer_cluster/network.tf b/examples/safer_cluster/network.tf new file mode 100644 index 0000000000..9bc38a9a3d --- /dev/null +++ b/examples/safer_cluster/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 1.4.0" + project_id = var.project_id + network_name = var.network + + subnets = [ + { + subnet_name = var.subnetwork + subnet_ip = var.subnetwork_cidr + subnet_region = var.region + }, + { + subnet_name = var.master_auth_subnetwork + subnet_ip = var.master_auth_subnetwork_cidr + subnet_region = var.region + }, + ] + + secondary_ranges = { + "${var.subnetwork}" = [ + { + range_name = var.ip_range_pods + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = var.ip_range_services + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 2ff9f4734b..8a9914cdf4 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -36,3 +36,13 @@ output "service_account" { value = module.gke.service_account } +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_names" { + description = "The names of the subnet being created" + value = module.gcp-network.subnets_names +} + diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index deb08f59e9..fc13c0a337 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -25,24 +25,39 @@ variable "cluster_name_suffix" { variable "region" { description = "The region to host the cluster in" + default = "us-central1" } variable "network" { description = "The VPC network to host the cluster in" + default = "gke-network" } variable "subnetwork" { description = "The subnetwork to host the cluster in" + default = "gke-subnet" +} +variable "subnetwork_cidr" { + description = "The cidr block for the subnetwork to host the cluster in" + default = "10.0.0.0/17" +} +variable "master_auth_subnetwork_cidr" { + description = "The cidr block for the subnetwork that has access to cluster master" + default = "10.60.0.0/17" +} +variable "master_auth_subnetwork" { + description = "The subnetwork that has access to cluster master" + default = "master-auth-subnet" } - variable "ip_range_pods" { description = "The secondary ip range to use for pods" + default = "ip-range-pods" } variable "ip_range_services" { description = "The secondary ip range to use for pods" + default = "ip-range-scv" } - variable "istio" { description = "Boolean to enable / disable Istio" default = true diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ad773982be..c4e3fcd41d 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -1,7 +1,5 @@ # Safer Cluster: How to setup a GKE Kubernetes cluster with reduced exposure -[TOC] - This module defines an opinionated setup of GKE cluster. We outline project configurations, cluster settings, and basic K8s objects that permit a safer-than-default configuration. @@ -37,7 +35,7 @@ are available for configuration, recommendations on their settings are documente control communication between applications. - We suggest to store user or business data persistently in managed storage - services that are inventories and controlled by centralized teams. + services that are inventoried and controlled by centralized teams. (e.g., GCP storage services within a GCP organization). - Storing user or business data securely requires satisfying a large set of @@ -47,7 +45,7 @@ are available for configuration, recommendations on their settings are documente ### Project Setup -We suggest a GKE setup composed of multiple projects to separate responsabilities +We suggest a GKE setup composed of multiple projects to separate responsibilities between cluster operators, which need to administer the cluster; and product developers, which mostly just want to deploy and debug applications. @@ -121,7 +119,7 @@ We suggest running different applications in different namespaces within the clu should be assigned to its own GCP service account to better define the Cloud IAM permissions required for the application. -If you are using more than 2 projects in your setup, you can consider creting +If you are using more than 2 projects in your setup, you can consider creating the service account in a different project to keep application and infrastructure separate. For example, service accounts could be created in each team's project, while the cluster runs in a centrally controlled project. @@ -239,6 +237,7 @@ For simplicity, we suggest using `roles/container.admin` and | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (required) | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 9069e22559..84c48b4a62 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -71,10 +71,6 @@ module "gke" { node_pools_taints = var.node_pools_taints node_pools_tags = var.node_pools_tags - // TODO(mmontan): we generally considered applying - // just the cloud-platoform scope and use Cloud IAM - // If we have Workload Identity, are there advantages - // in restricting scopes even more? node_pools_oauth_scopes = var.node_pools_oauth_scopes stub_domains = var.stub_domains @@ -92,13 +88,10 @@ module "gke" { // All applications shuold run with an identity defined via Workload Identity anyway. // - Use a service account passed as a parameter to the module, in case the user // wants to maintain control of their service accounts. - create_service_account = var.compute_engine_service_account == "" ? false : true + create_service_account = var.compute_engine_service_account == "" ? true : false service_account = var.compute_engine_service_account - - // TODO(mmontan): define a registry_project parameter in the private_beta_cluster, - // so that we can give GCS permissions to the service account on a project - // that hosts only container-images and not data. - grant_registry_access = true + registry_project_id = var.registry_project_id + grant_registry_access = true // Basic Auth disabled basic_auth_username = "" @@ -133,7 +126,7 @@ module "gke" { enable_binary_authorization = true // Define PodSecurityPolicies for differnet applications. - // TODO(mmontan): link to a couple of policies. + // Example: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#example pod_security_policy_config = [{ "enabled" = true }] @@ -144,7 +137,7 @@ module "gke" { // Sandbox can also provide increased protection in other cases, at some performance cost. sandbox_enabled = var.sandbox_enabled - // TODO(mmontan): investigate whether this should be a recommended setting + // Intranode Visibility enables you to capture flow logs for traffic between pods and create FW rules that apply to traffic between pods. enable_intranode_visibility = var.enable_intranode_visibility enable_vertical_pod_autoscaling = var.enable_vertical_pod_autoscaling diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 71b4ba88ef..1b9d2d02d8 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -208,8 +208,11 @@ variable "grant_registry_access" { default = false } -// TODO(mmontan): allow specifying which project to use -// for reading images. +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} variable "service_account" { type = string diff --git a/test/ci/safer-cluster.yml b/test/ci/safer-cluster.yml deleted file mode 100644 index ef3c896b29..0000000000 --- a/test/ci/safer-cluster.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- - -platform: linux - -inputs: -- name: pull-request - path: terraform-google-kubernetes-engine - -run: - path: make - args: ['test_integration'] - dir: terraform-google-kubernetes-engine - -params: - SUITE: "safer-cluster-local" - COMPUTE_ENGINE_SERVICE_ACCOUNT: "" - REGION: "us-east4" - ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index c9d659ba67..a130769b42 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -18,11 +18,6 @@ module "example" { source = "../../../examples/safer_cluster" project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" region = var.region - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account } diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf deleted file mode 100644 index e1292eae3b..0000000000 --- a/test/fixtures/safer_cluster/network.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} - -provider "google" { - project = var.project_id -} - -resource "google_compute_network" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = var.region - network = google_compute_network.main.self_link - - secondary_ip_range { - range_name = "cft-gke-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "cft-gke-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} - diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf deleted file mode 120000 index 726bdc722f..0000000000 --- a/test/fixtures/safer_cluster/outputs.tf +++ /dev/null @@ -1 +0,0 @@ -../shared/outputs.tf \ No newline at end of file diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf new file mode 100644 index 0000000000..86b0648afd --- /dev/null +++ b/test/fixtures/safer_cluster/outputs.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} +output "location" { + value = module.example.location +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} From fb3861210597273bfbdcf3732391f208d185b6bb Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Wed, 13 Nov 2019 15:32:05 -0600 Subject: [PATCH 013/110] fix lint --- examples/safer_cluster/README.md | 17 ----------------- examples/safer_cluster/outputs.tf | 4 ---- 2 files changed, 21 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index db76f2edb8..ba26e59314 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -10,7 +10,6 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -<<<<<<< HEAD | ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | | ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | | istio | Boolean to enable / disable Istio | string | `"true"` | no | @@ -22,16 +21,6 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | region | The region to host the cluster in | string | `"us-central1"` | no | | subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | | subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | -======= -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | -| istio | Boolean to enable / disable Istio | string | `"true"` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | n/a | yes | -| project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 ## Outputs @@ -46,17 +35,11 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | The master Kubernetes version | | network | | -<<<<<<< HEAD | network\_name | The name of the VPC being created | | project\_id | | | region | | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | -======= -| project\_id | | -| region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 6b42b830c7..eee2f4fbf2 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -36,7 +36,6 @@ output "service_account" { value = module.gke.service_account } -<<<<<<< HEAD output "network_name" { description = "The name of the VPC being created" value = module.gcp-network.network_name @@ -46,6 +45,3 @@ output "subnet_names" { description = "The names of the subnet being created" value = module.gcp-network.subnets_names } - -======= ->>>>>>> 6f7de1836f077258c609f0d7129a60bfadc4f8b1 From 62418d0af76e23e3854f4aac56fb5f68735f4f1f Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Wed, 13 Nov 2019 18:11:08 -0800 Subject: [PATCH 014/110] Example to create private cluster with node pool specifications along with oauth scopes --- .../README.md | 41 ++++++ .../main.tf | 110 ++++++++++++++++ .../network.tf | 41 ++++++ .../outputs.tf | 117 ++++++++++++++++++ .../provider.tf | 24 ++++ .../variables.tf | 50 ++++++++ 6 files changed, 383 insertions(+) create mode 100644 examples/regional_private_node_pool_oauth_scopes/README.md create mode 100644 examples/regional_private_node_pool_oauth_scopes/main.tf create mode 100644 examples/regional_private_node_pool_oauth_scopes/network.tf create mode 100644 examples/regional_private_node_pool_oauth_scopes/outputs.tf create mode 100644 examples/regional_private_node_pool_oauth_scopes/provider.tf create mode 100644 examples/regional_private_node_pool_oauth_scopes/variables.tf diff --git a/examples/regional_private_node_pool_oauth_scopes/README.md b/examples/regional_private_node_pool_oauth_scopes/README.md new file mode 100644 index 0000000000..2fd7b5139a --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/README.md @@ -0,0 +1,41 @@ +# Regional Private Cluster with node pool and oauth scopes + +This example illustrates how to create a private cluster with node pool specifications, oauth scopes along with required network and subnet creation. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| cluster\_name | Name of the cluster | string | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_name | Cluster name | +| cluster\_type | Cluster type - Regional or Zonal | +| location | Cluster location | +| region | Cluster region | +| zones | List of zones in which the cluster resides | +| min\_master\_version | Minimum master kubernetes version | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| service\_account | The default service account used for running nodes. | +| network | Network module output | + + + +To provision this example, run the following from within this directory: +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf new file mode 100644 index 0000000000..e6947f35ad --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -0,0 +1,110 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +data "google_compute_subnetwork" "subnetwork" { + name = module.gke-network.subnets_names[0] + project = var.project_id + region = var.region +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + project_id = var.project_id + name = var.cluster_name + region = var.region + regional = true + network = module.gke-network.network_name + subnetwork = module.gke-network.subnets_names[0] + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.16/28" + network_policy = true + horizontal_pod_autoscaling = true + service_account = "create" + remove_default_node_pool = true + disable_legacy_metadata_endpoints = true + +master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] + + node_pools = [ + { + name = "my-node-pool" + machine_type = "n1-standard-1" + min_count = 1 + max_count = 1 + disk_size_gb = 100 + disk_type = "pd-ssd" + image_type = "COS" + auto_repair = true + auto_upgrade = false + preemptible = false + initial_node_count = 1 + }, + ] + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/servicecontrol", + ] + + my-node-pool = [ + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/servicecontrol", + ] + } + + node_pools_labels = { + + all = { + + } + my-node-pool = { + + } + } + + node_pools_metadata = { + all = {} + + my-node-pool = {} + + } + + node_pools_tags = { + all = [] + + my-node-pool = [] + + } +} diff --git a/examples/regional_private_node_pool_oauth_scopes/network.tf b/examples/regional_private_node_pool_oauth_scopes/network.tf new file mode 100644 index 0000000000..8295d9f49d --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/network.tf @@ -0,0 +1,41 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +module "gke-network" { + source = "terraform-google-modules/network/google" + project_id = var.project_id + network_name = var.network + + subnets = [ + { + subnet_name = var.subnet + subnet_ip = "10.0.0.0/24" + subnet_region = var.region + }, + ] + + secondary_ranges = { + "${var.subnet}" = [ + { + range_name = var.ip_range_pods + ip_cidr_range = "10.1.0.0/16" + }, + { + range_name = var.ip_range_services + ip_cidr_range = "10.2.0.0/20" + }, + ]} +} diff --git a/examples/regional_private_node_pool_oauth_scopes/outputs.tf b/examples/regional_private_node_pool_oauth_scopes/outputs.tf new file mode 100644 index 0000000000..765205ff39 --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/outputs.tf @@ -0,0 +1,117 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +output "cluster_name" { + description = "Cluster name" + value = "${module.gke.name}" +} + +output "type" { + description = "Cluster type (regional / zonal)" + value = "${module.gke.type}" +} + +output "location" { + description = "Cluster location (region if regional cluster, zone if zonal cluster)" + value = "${module.gke.location}" +} + +output "region" { + description = "Cluster region" + value = "${module.gke.region}" +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = "${module.gke.zones}" +} + +output "endpoint" { + sensitive = true + description = "Cluster endpoint" + value = "${module.gke.endpoint}" +} + +output "min_master_version" { + description = "Minimum master kubernetes version" + value = "${module.gke.min_master_version}" +} + +output "logging_service" { + description = "Logging service used" + value = "${module.gke.logging_service}" +} + +output "monitoring_service" { + description = "Monitoring service used" + value = "${module.gke.monitoring_service}" +} + +output "master_authorized_networks_config" { + description = "Networks from which access to master is permitted" + value = "${module.gke.master_authorized_networks_config}" +} + +output "master_version" { + description = "Current master kubernetes version" + value = "${module.gke.master_version}" +} + +output "ca_certificate" { + sensitive = true + description = "Cluster ca certificate (base64 encoded)" + value = "${module.gke.ca_certificate}" +} + +output "network_policy_enabled" { + description = "Whether network policy enabled" + value = "${module.gke.network_policy_enabled}" +} + +output "http_load_balancing_enabled" { + description = "Whether http load balancing enabled" + value = "${module.gke.http_load_balancing_enabled}" +} + +output "horizontal_pod_autoscaling_enabled" { + description = "Whether horizontal pod autoscaling enabled" + value = "${module.gke.horizontal_pod_autoscaling_enabled}" +} + +output "kubernetes_dashboard_enabled" { + description = "Whether kubernetes dashboard enabled" + value = "${module.gke.kubernetes_dashboard_enabled}" +} + +output "node_pools_names" { + description = "List of node pools names" + value = "${module.gke.node_pools_names}" +} + +output "node_pools_versions" { + description = "List of node pools versions" + value = "${module.gke.node_pools_versions}" +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = "${module.gke.service_account}" +} + +output "network_module" { + description = "network module output" + value = module.gke-network +} diff --git a/examples/regional_private_node_pool_oauth_scopes/provider.tf b/examples/regional_private_node_pool_oauth_scopes/provider.tf new file mode 100644 index 0000000000..7fc7ee64bb --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/provider.tf @@ -0,0 +1,24 @@ +/* +Copyright 2019 Google LLC +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + https://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +provider "google" { + version = "2.18.0" + project = var.project_id + region = var.region +} + +provider "google-beta" { + version = "2.18.0" + project = var.project_id + region = var.region +} diff --git a/examples/regional_private_node_pool_oauth_scopes/variables.tf b/examples/regional_private_node_pool_oauth_scopes/variables.tf new file mode 100644 index 0000000000..4c3c5a907d --- /dev/null +++ b/examples/regional_private_node_pool_oauth_scopes/variables.tf @@ -0,0 +1,50 @@ +/* +Copyright 2019 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +variable "cluster_name" { + description = "Name of the cluster" + default = "test-cluster" +} + +variable "project_id" { + description = "The project ID to host the cluster in" + default = "alekhya-lakkadi" +} + +variable "region" { + description = "The region to host the cluster in" + default = "us-west1" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" + default = "cluster-ip-range-pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for pods" + default = "cluster-ip-range-services" +} + +variable "network" { + description = "The VPC network name to host the cluster in" + default = "my-network" +} + +variable "subnet" { + description = "The subnetwork name to host the cluster in" + default = "my-subnet" +} From 9370965e3a6b95039b3728514ba712639ea0c75c Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Thu, 14 Nov 2019 13:15:02 -0800 Subject: [PATCH 015/110] Terraform fmt and default values are removed --- .../main.tf | 14 +++++++------- .../network.tf | 2 +- .../variables.tf | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index e6947f35ad..12e7a33511 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -39,7 +39,7 @@ module "gke" { remove_default_node_pool = true disable_legacy_metadata_endpoints = true -master_authorized_networks_config = [ + master_authorized_networks_config = [ { cidr_blocks = [ { @@ -68,12 +68,12 @@ master_authorized_networks_config = [ node_pools_oauth_scopes = { all = [ - "https://www.googleapis.com/auth/trace.append", - "https://www.googleapis.com/auth/service.management.readonly", - "https://www.googleapis.com/auth/monitoring", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/servicecontrol", - ] + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/servicecontrol", + ] my-node-pool = [ "https://www.googleapis.com/auth/trace.append", diff --git a/examples/regional_private_node_pool_oauth_scopes/network.tf b/examples/regional_private_node_pool_oauth_scopes/network.tf index 8295d9f49d..e42f7099b2 100644 --- a/examples/regional_private_node_pool_oauth_scopes/network.tf +++ b/examples/regional_private_node_pool_oauth_scopes/network.tf @@ -37,5 +37,5 @@ module "gke-network" { range_name = var.ip_range_services ip_cidr_range = "10.2.0.0/20" }, - ]} + ] } } diff --git a/examples/regional_private_node_pool_oauth_scopes/variables.tf b/examples/regional_private_node_pool_oauth_scopes/variables.tf index 4c3c5a907d..a7dd2b323f 100644 --- a/examples/regional_private_node_pool_oauth_scopes/variables.tf +++ b/examples/regional_private_node_pool_oauth_scopes/variables.tf @@ -16,17 +16,15 @@ limitations under the License. variable "cluster_name" { description = "Name of the cluster" - default = "test-cluster" + default = "test-cluster" } variable "project_id" { description = "The project ID to host the cluster in" - default = "alekhya-lakkadi" } variable "region" { description = "The region to host the cluster in" - default = "us-west1" } variable "ip_range_pods" { @@ -41,10 +39,8 @@ variable "ip_range_services" { variable "network" { description = "The VPC network name to host the cluster in" - default = "my-network" } variable "subnet" { description = "The subnetwork name to host the cluster in" - default = "my-subnet" } From 1ca8f82b4947bc6b7f1eb6352a32cf180def00be Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Thu, 14 Nov 2019 15:16:38 -0800 Subject: [PATCH 016/110] Fixed lint issues --- .../README.md | 28 ++++++++++------ .../main.tf | 32 +++++++++--------- .../network.tf | 30 ++++++++--------- .../outputs.tf | 30 ++++++++--------- .../provider.tf | 27 ++++++++------- .../variables.tf | 33 +++++++++---------- 6 files changed, 94 insertions(+), 86 deletions(-) diff --git a/examples/regional_private_node_pool_oauth_scopes/README.md b/examples/regional_private_node_pool_oauth_scopes/README.md index 2fd7b5139a..031f4709e2 100644 --- a/examples/regional_private_node_pool_oauth_scopes/README.md +++ b/examples/regional_private_node_pool_oauth_scopes/README.md @@ -7,30 +7,38 @@ This example illustrates how to create a private cluster with node pool specific | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| project\_id | The project ID to host the cluster in | string | n/a | yes | | cluster\_name | Name of the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | -| network | The VPC network to host the cluster in | string | n/a | yes | +| network | The VPC network name to host the cluster in | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | -| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| subnet | The subnetwork name to host the cluster in | string | n/a | yes | ## Outputs | Name | Description | |------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_name | Cluster name | -| cluster\_type | Cluster type - Regional or Zonal | -| location | Cluster location | -| region | Cluster region | -| zones | List of zones in which the cluster resides | -| min\_master\_version | Minimum master kubernetes version | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | | master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| network\_module | network module output | +| network\_policy\_enabled | Whether network policy enabled | | node\_pools\_names | List of node pools names | | node\_pools\_versions | List of node pools versions | -| service\_account | The default service account used for running nodes. | -| network | Network module output | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index 12e7a33511..29770985cb 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -1,18 +1,18 @@ -/* -Copyright 2019 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ data "google_compute_subnetwork" "subnetwork" { name = module.gke-network.subnets_names[0] @@ -21,7 +21,7 @@ data "google_compute_subnetwork" "subnetwork" { } module "gke" { - source = "terraform-google-modules/kubernetes-engine/google//modules/private-cluster" + source = "../../modules/private-cluster" project_id = var.project_id name = var.cluster_name region = var.region diff --git a/examples/regional_private_node_pool_oauth_scopes/network.tf b/examples/regional_private_node_pool_oauth_scopes/network.tf index e42f7099b2..624236f63c 100644 --- a/examples/regional_private_node_pool_oauth_scopes/network.tf +++ b/examples/regional_private_node_pool_oauth_scopes/network.tf @@ -1,18 +1,18 @@ -/* -Copyright 2019 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ module "gke-network" { source = "terraform-google-modules/network/google" diff --git a/examples/regional_private_node_pool_oauth_scopes/outputs.tf b/examples/regional_private_node_pool_oauth_scopes/outputs.tf index 765205ff39..464ed303c3 100644 --- a/examples/regional_private_node_pool_oauth_scopes/outputs.tf +++ b/examples/regional_private_node_pool_oauth_scopes/outputs.tf @@ -1,18 +1,18 @@ -/* -Copyright 2018 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ output "cluster_name" { description = "Cluster name" diff --git a/examples/regional_private_node_pool_oauth_scopes/provider.tf b/examples/regional_private_node_pool_oauth_scopes/provider.tf index 7fc7ee64bb..4317f93e11 100644 --- a/examples/regional_private_node_pool_oauth_scopes/provider.tf +++ b/examples/regional_private_node_pool_oauth_scopes/provider.tf @@ -1,15 +1,18 @@ -/* -Copyright 2019 Google LLC -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ provider "google" { version = "2.18.0" diff --git a/examples/regional_private_node_pool_oauth_scopes/variables.tf b/examples/regional_private_node_pool_oauth_scopes/variables.tf index a7dd2b323f..07352a1a0b 100644 --- a/examples/regional_private_node_pool_oauth_scopes/variables.tf +++ b/examples/regional_private_node_pool_oauth_scopes/variables.tf @@ -1,22 +1,21 @@ -/* -Copyright 2019 Google LLC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ variable "cluster_name" { description = "Name of the cluster" - default = "test-cluster" } variable "project_id" { @@ -29,12 +28,10 @@ variable "region" { variable "ip_range_pods" { description = "The secondary ip range to use for pods" - default = "cluster-ip-range-pods" } variable "ip_range_services" { description = "The secondary ip range to use for pods" - default = "cluster-ip-range-services" } variable "network" { From 1a5eb6354dc0281e7cf4c1a0f4e2159afed08e48 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 18 Nov 2019 08:19:13 -0600 Subject: [PATCH 017/110] add types, fix desc --- examples/safer_cluster/README.md | 4 +-- examples/safer_cluster/main.tf | 8 ++--- examples/safer_cluster/variables.tf | 14 ++++++++ modules/safer-cluster/README.md | 12 +++---- modules/safer-cluster/main.tf | 7 ++-- modules/safer-cluster/variables.tf | 12 +++---- test/fixtures/safer_cluster/network.tf | 48 -------------------------- 7 files changed, 33 insertions(+), 72 deletions(-) delete mode 100644 test/fixtures/safer_cluster/network.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index ba26e59314..845b263d13 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -7,12 +7,12 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | +| cloudrun | Boolean to enable / disable CloudRun | bool | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | | ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | -| istio | Boolean to enable / disable Istio | string | `"true"` | no | +| istio | Boolean to enable / disable Istio | bool | `"true"` | no | | master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | | master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index e865c9c2f9..bcae52aac9 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -18,14 +18,12 @@ locals { cluster_type = "safer-cluster" } -provider "google-beta" { +provider "google" { version = "~> 2.18.0" } -data "google_compute_subnetwork" "subnetwork" { - name = var.subnetwork - project = var.project_id - region = var.region +provider "google-beta" { + version = "~> 2.18.0" } module "gke" { diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index fc13c0a337..6b726747ca 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -15,64 +15,78 @@ */ variable "project_id" { + type = string description = "The project ID to host the cluster in" } variable "cluster_name_suffix" { + type = string description = "A suffix to append to the default cluster name" default = "" } variable "region" { + type = string description = "The region to host the cluster in" default = "us-central1" } variable "network" { + type = string description = "The VPC network to host the cluster in" default = "gke-network" } variable "subnetwork" { + type = string description = "The subnetwork to host the cluster in" default = "gke-subnet" } variable "subnetwork_cidr" { + type = string description = "The cidr block for the subnetwork to host the cluster in" default = "10.0.0.0/17" } variable "master_auth_subnetwork_cidr" { + type = string description = "The cidr block for the subnetwork that has access to cluster master" default = "10.60.0.0/17" } variable "master_auth_subnetwork" { + type = string description = "The subnetwork that has access to cluster master" default = "master-auth-subnet" } variable "ip_range_pods" { + type = string description = "The secondary ip range to use for pods" default = "ip-range-pods" } variable "ip_range_services" { + type = string description = "The secondary ip range to use for pods" default = "ip-range-scv" } variable "istio" { + type = bool description = "Boolean to enable / disable Istio" default = true } variable "cloudrun" { + type = bool description = "Boolean to enable / disable CloudRun" default = true } variable "master_ipv4_cidr_block" { + type = string description = "The IP range in CIDR notation to use for the hosted master network" default = "172.16.0.0/28" } variable "compute_engine_service_account" { + type = string description = "Service account to associate to the nodes in the cluster" } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index c4e3fcd41d..4b51d4d5e3 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -224,8 +224,8 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| name | The name of the cluster | string | n/a | yes | +| network | The VPC network to host the cluster in | string | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | @@ -234,17 +234,17 @@ For simplicity, we suggest using `roles/container.admin` and | node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | | node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| project\_id | The project ID to host the cluster in | string | n/a | yes | +| region | The region to host the cluster in | string | n/a | yes | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | +| zones | The zones to host the cluster in | list(string) | `` | no | ## Outputs diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 84c48b4a62..76c40b3ebe 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -62,11 +62,8 @@ module "gke" { // destroying the cluster. remove_default_node_pool = true - node_pools = var.node_pools - node_pools_labels = var.node_pools_labels - - // TODO(mmontan): check whether we need to restrict these - // settings. + node_pools = var.node_pools + node_pools_labels = var.node_pools_labels node_pools_metadata = var.node_pools_metadata node_pools_taints = var.node_pools_taints node_pools_tags = var.node_pools_tags diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 1b9d2d02d8..fe47be7be6 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -18,12 +18,12 @@ variable "project_id" { type = string - description = "The project ID to host the cluster in (required)" + description = "The project ID to host the cluster in" } variable "name" { type = string - description = "The name of the cluster (required)" + description = "The name of the cluster" } variable "description" { @@ -40,18 +40,18 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in" } variable "zones" { type = list(string) - description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + description = "The zones to host the cluster in" default = [] } variable "network" { type = string - description = "The VPC network to host the cluster in (required)" + description = "The VPC network to host the cluster in" } variable "network_project_id" { @@ -62,7 +62,7 @@ variable "network_project_id" { variable "subnetwork" { type = string - description = "The subnetwork to host the cluster in (required)" + description = "The subnetwork to host the cluster in" } variable "kubernetes_version" { diff --git a/test/fixtures/safer_cluster/network.tf b/test/fixtures/safer_cluster/network.tf deleted file mode 100644 index e1292eae3b..0000000000 --- a/test/fixtures/safer_cluster/network.tf +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Copyright 2018 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_string" "suffix" { - length = 4 - special = false - upper = false -} - -provider "google" { - project = var.project_id -} - -resource "google_compute_network" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "main" { - name = "cft-gke-test-${random_string.suffix.result}" - ip_cidr_range = "10.0.0.0/17" - region = var.region - network = google_compute_network.main.self_link - - secondary_ip_range { - range_name = "cft-gke-test-pods-${random_string.suffix.result}" - ip_cidr_range = "192.168.0.0/18" - } - - secondary_ip_range { - range_name = "cft-gke-test-services-${random_string.suffix.result}" - ip_cidr_range = "192.168.64.0/18" - } -} - From ac3b601e00af31c3b318dae30be07305d39eed9b Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Thu, 24 Oct 2019 13:29:20 -0500 Subject: [PATCH 018/110] Updated node_pools variable documentation --- autogen/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/autogen/README.md b/autogen/README.md index 3efe785ff0..1d1053a4af 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -185,6 +185,22 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools Variable +The node_pools variable takes the following parameters: + +name - (Optional) The name of the node pool. If left blank, Terraform will auto-generate a unique name +min_count - (Required) Minimum number of nodes in the NodePool. Must be >=0 and <= max_count +max_count - (Required) Maximum number of nodes in the NodePool. Must be >= min_count +machine_type - (Optional) The name of a Google Compute Engine machine type. Defaults to n1-standard-1 +disk_size_gb - (Optional) Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB +disk_type - (Optional) Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' +image_type - (Optional) The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool +preemptible - (Optional) A boolean that represents whether or not the underlying node VMs are preemptible +service_account - (Optional) The service account to be used by the Node VMs. If not specified, the "default" service account is used +initial_node_count - (Optional) The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource +auto_repair - (Optional) Whether the nodes will be automatically repaired +auto_upgrade - (Optional) Whether the nodes will be automatically upgraded + ## File structure The project has the following folders and files: From f6a1fe73fac13b6cb8ca30da9b66d58c6a642786 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Thu, 24 Oct 2019 16:02:32 -0500 Subject: [PATCH 019/110] Added tables to the documentation --- autogen/README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 1d1053a4af..75027a6e35 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -188,18 +188,20 @@ In order to operate with the Service Account you must activate the following API ## node_pools Variable The node_pools variable takes the following parameters: -name - (Optional) The name of the node pool. If left blank, Terraform will auto-generate a unique name -min_count - (Required) Minimum number of nodes in the NodePool. Must be >=0 and <= max_count -max_count - (Required) Maximum number of nodes in the NodePool. Must be >= min_count -machine_type - (Optional) The name of a Google Compute Engine machine type. Defaults to n1-standard-1 -disk_size_gb - (Optional) Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB -disk_type - (Optional) Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' -image_type - (Optional) The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool -preemptible - (Optional) A boolean that represents whether or not the underlying node VMs are preemptible -service_account - (Optional) The service account to be used by the Node VMs. If not specified, the "default" service account is used -initial_node_count - (Optional) The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource -auto_repair - (Optional) Whether the nodes will be automatically repaired -auto_upgrade - (Optional) Whether the nodes will be automatically upgraded +| Name | Description | Requirement | +| --- | --- | --- | +| name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | Required | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | Required | +| machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | Optional | +| service_account | The service account to be used by the Node VMs. If not specified, the "default" service account is used | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | Optional | +| auto_repair | Whether the nodes will be automatically repaired | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | Optional | ## File structure The project has the following folders and files: From 271fc6abfd7032ed520177cf27793ef778976af4 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Wed, 30 Oct 2019 17:44:28 -0500 Subject: [PATCH 020/110] Made changes to the variables --- autogen/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 75027a6e35..3025ab2753 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -185,14 +185,14 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools Variable +## node_pools variable The node_pools variable takes the following parameters: | Name | Description | Requirement | | --- | --- | --- | | name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | Required | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | Required | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | | machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | From 4600b989695aec5d4661c18b31f4282e780d96bd Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 10:55:09 -0500 Subject: [PATCH 021/110] Added the autoscaling property --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index 3025ab2753..122be74d42 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -191,6 +191,7 @@ The node_pools variable takes the following parameters: | Name | Description | Requirement | | --- | --- | --- | | name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage. Default value is true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | | machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | From 56353d6956b226b4b17faa9e42e45220773069b3 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 12:48:31 -0500 Subject: [PATCH 022/110] Default column added --- autogen/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 122be74d42..3777608c19 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -188,21 +188,21 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Requirement | -| --- | --- | --- | -| name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage. Default value is true | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | Optional | -| service_account | The service account to be used by the Node VMs. If not specified, the "default" service account is used | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | Optional | -| auto_repair | Whether the nodes will be automatically repaired | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | Optional | +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| name | The name of the node pool | "" | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-1 | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | "" | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | ## File structure The project has the following folders and files: From 45f6ac8734c566ebc86c719ded59919462e25477 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 12:50:29 -0500 Subject: [PATCH 023/110] Default column added --- autogen/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 3777608c19..6e5329294e 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -190,7 +190,7 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | -| name | The name of the node pool | "" | Optional | +| name | The name of the node pool | " " | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | @@ -199,7 +199,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | "" | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | From b8b4e7c884b8d64c3175c5f37afffd647cb6b6c1 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 4 Nov 2019 15:18:05 -0600 Subject: [PATCH 024/110] Made changes to the table --- autogen/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 6e5329294e..a4b8c4359f 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -194,7 +194,7 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-1 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | @@ -202,7 +202,7 @@ The node_pools variable takes the following parameters: | service_account | The service account to be used by the Node VMs | " " | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | false | Optional | ## File structure The project has the following folders and files: From 82217de3af7281be316625bbc81fc237c10320e4 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 4 Nov 2019 23:13:27 -0600 Subject: [PATCH 025/110] Added node_locations --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index a4b8c4359f..422ee66440 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -192,6 +192,7 @@ The node_pools variable takes the following parameters: | --- | --- | --- | --- | | name | The name of the node pool | " " | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | From 9c5983d2a4b09db585c8975f05920f54c3e63853 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 8 Nov 2019 23:41:26 -0600 Subject: [PATCH 026/110] Updated auto upgrade --- autogen/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 422ee66440..fde511eee1 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -190,20 +190,20 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | -| name | The name of the node pool | " " | Optional | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | false | Optional | ## File structure The project has the following folders and files: From c8bcfaa9346f5b79a2b658abc9fdb089867c8644 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 10:43:50 -0600 Subject: [PATCH 027/110] node count added --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index fde511eee1..e61fe093c3 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -201,6 +201,7 @@ The node_pools variable takes the following parameters: | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | From f466f0dc3ab7607cf5b5d510e5f27a436c2c34a2 Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Mon, 18 Nov 2019 10:32:49 -0800 Subject: [PATCH 028/110] doubled the timeout in .kitchen.yml --- .kitchen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kitchen.yml b/.kitchen.yml index 39faa2e1e3..ab08aebfd2 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -15,7 +15,7 @@ --- driver: name: "terraform" - command_timeout: 1800 + command_timeout: 3600 provisioner: name: "terraform" From fdfa88c9e37c32e0246dc647f3830d8e63f4eaca Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Mon, 18 Nov 2019 11:46:31 -0800 Subject: [PATCH 029/110] changing the timeout value --- .kitchen.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kitchen.yml b/.kitchen.yml index ab08aebfd2..39faa2e1e3 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -15,7 +15,7 @@ --- driver: name: "terraform" - command_timeout: 3600 + command_timeout: 1800 provisioner: name: "terraform" From b18a5df00a829f80d27bf588bca1fba847dddcd0 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:39:17 -0600 Subject: [PATCH 030/110] Fixed Errors --- README.md | 96 +++----------- .../README.md | 123 +++--------------- modules/beta-private-cluster/README.md | 123 +++--------------- modules/beta-public-cluster/README.md | 119 +++-------------- .../private-cluster-update-variant/README.md | 100 +++----------- modules/private-cluster/README.md | 100 +++----------- 6 files changed, 126 insertions(+), 535 deletions(-) diff --git a/README.md b/README.md index 15f6aff13b..e91e987e2b 100644 --- a/README.md +++ b/README.md @@ -125,81 +125,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -239,6 +164,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d2b5197726..64cba85d93 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -133,108 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity namespace | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -274,6 +172,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 96ad9abf5a..c4e644bcdc 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -133,108 +133,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity namespace | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -274,6 +172,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fcf78cccc..c69a27cee1 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -128,104 +128,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| identity\_namespace | Workload Identity namespace | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| release\_channel | The release channel of this cluster | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -265,6 +167,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index fa9cdb8852..17e50565cf 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -248,6 +169,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 5465544b82..f61be13402 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -130,85 +130,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -248,6 +169,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: From 2ae6c27b746abfd6a10ae07a968881d59b7e3a66 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:42:44 -0600 Subject: [PATCH 031/110] Fixed Errors --- README.md | 75 +++++++++++++ .../README.md | 102 ++++++++++++++++++ modules/beta-private-cluster/README.md | 102 ++++++++++++++++++ modules/beta-public-cluster/README.md | 98 +++++++++++++++++ .../private-cluster-update-variant/README.md | 79 ++++++++++++++ modules/private-cluster/README.md | 79 ++++++++++++++ 6 files changed, 535 insertions(+) diff --git a/README.md b/README.md index e91e987e2b..9848edcf72 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,81 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 64cba85d93..4add734de5 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -133,6 +133,108 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index c4e644bcdc..d8ed379724 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -133,6 +133,108 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index c69a27cee1..e051704a5c 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -128,6 +128,104 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | +| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | +| enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | bool | `"false"` | no | +| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| identity\_namespace | Workload Identity namespace | string | `""` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| istio | (Beta) Enable Istio addon | string | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| release\_channel | (Beta) The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `UNSPECIFIED`. | string | `"null"` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | +| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| cloudrun\_enabled | Whether CloudRun enabled | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| identity\_namespace | Workload Identity namespace | +| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | +| istio\_enabled | Whether Istio is enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| pod\_security\_policy\_enabled | Whether pod security policy is enabled | +| region | Cluster region | +| release\_channel | The release channel of this cluster | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 17e50565cf..00cd07588d 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index f61be13402..2a30835037 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -130,6 +130,85 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|:----:|:-----:|:-----:| +| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | +| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | +| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | +| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | +| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | +| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | +| description | The description of the cluster | string | `""` | no | +| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | +| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | +| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | +| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | +| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | +| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | +| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | +| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | +| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | +| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | +| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | +| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | +| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | +| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | +| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | +| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | +| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | +| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | +| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | +| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | +| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | +| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | Cluster ca certificate (base64 encoded) | +| endpoint | Cluster endpoint | +| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | +| http\_load\_balancing\_enabled | Whether http load balancing enabled | +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +| location | Cluster location (region if regional cluster, zone if zonal cluster) | +| logging\_service | Logging service used | +| master\_authorized\_networks\_config | Networks from which access to master is permitted | +| master\_version | Current master kubernetes version | +| min\_master\_version | Minimum master kubernetes version | +| monitoring\_service | Monitoring service used | +| name | Cluster name | +| network\_policy\_enabled | Whether network policy enabled | +| node\_pools\_names | List of node pools names | +| node\_pools\_versions | List of node pools versions | +| region | Cluster region | +| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| type | Cluster type (regional / zonal) | +| zones | List of zones in which the cluster resides | + ## Requirements From 4aa670fbf5b6146c68dd1a3cd0f9882a773aeeb8 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:48:46 -0600 Subject: [PATCH 032/110] Fixed Errors --- README.md | 2 +- autogen/README.md | 2 +- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-public-cluster/README.md | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9848edcf72..7ca60f7541 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/autogen/README.md b/autogen/README.md index e61fe093c3..ca0215c920 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -188,7 +188,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 4add734de5..ee53101dff 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -277,7 +277,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index d8ed379724..0a72479ad0 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -277,7 +277,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index e051704a5c..a0ed11ebbb 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -268,7 +268,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 00cd07588d..0ebff66933 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -251,7 +251,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 2a30835037..257515ef7b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -251,7 +251,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | From 9b821bf7534d72cc0b83fe45548e5127ece59571 Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Tue, 19 Nov 2019 14:44:59 -0800 Subject: [PATCH 033/110] addressed comments made for regional_private_node_pool example --- .../README.md | 8 +--- .../main.tf | 16 ++----- .../network.tf | 13 ++--- .../outputs.tf | 48 +++++++++++-------- .../provider.tf | 4 -- .../variables.tf | 24 ---------- 6 files changed, 43 insertions(+), 70 deletions(-) diff --git a/examples/regional_private_node_pool_oauth_scopes/README.md b/examples/regional_private_node_pool_oauth_scopes/README.md index 031f4709e2..3ae6a8ca77 100644 --- a/examples/regional_private_node_pool_oauth_scopes/README.md +++ b/examples/regional_private_node_pool_oauth_scopes/README.md @@ -7,13 +7,7 @@ This example illustrates how to create a private cluster with node pool specific | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cluster\_name | Name of the cluster | string | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | -| network | The VPC network name to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | -| region | The region to host the cluster in | string | n/a | yes | -| subnet | The subnetwork name to host the cluster in | string | n/a | yes | ## Outputs @@ -37,6 +31,8 @@ This example illustrates how to create a private cluster with node pool specific | node\_pools\_versions | List of node pools versions | | region | Cluster region | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| subnets\_ips | The IP and cidrs of the subnets being created | +| subnets\_secondary\_ranges | The secondary ranges associated with these subnets | | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index 29770985cb..e39e3299f6 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -14,22 +14,16 @@ * limitations under the License. */ -data "google_compute_subnetwork" "subnetwork" { - name = module.gke-network.subnets_names[0] - project = var.project_id - region = var.region -} - module "gke" { source = "../../modules/private-cluster" project_id = var.project_id - name = var.cluster_name - region = var.region + name = "random-test-cluster" + region = "us-west1" regional = true network = module.gke-network.network_name subnetwork = module.gke-network.subnets_names[0] - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services + ip_range_pods = module.gke-network.subnets_secondary_ranges[0].*.range_name[0] + ip_range_services = module.gke-network.subnets_secondary_ranges[0].*.range_name[1] enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.16/28" @@ -43,7 +37,7 @@ module "gke" { { cidr_blocks = [ { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + cidr_block = module.gke-network.subnets_ips[0] display_name = "VPC" }, ] diff --git a/examples/regional_private_node_pool_oauth_scopes/network.tf b/examples/regional_private_node_pool_oauth_scopes/network.tf index 624236f63c..2d15f20c2b 100644 --- a/examples/regional_private_node_pool_oauth_scopes/network.tf +++ b/examples/regional_private_node_pool_oauth_scopes/network.tf @@ -16,25 +16,26 @@ module "gke-network" { source = "terraform-google-modules/network/google" + version = "~> 1.5" project_id = var.project_id - network_name = var.network + network_name = "random-gke-network" subnets = [ { - subnet_name = var.subnet + subnet_name = "random-gke-subnet" subnet_ip = "10.0.0.0/24" - subnet_region = var.region + subnet_region = "us-west1" }, ] secondary_ranges = { - "${var.subnet}" = [ + "random-gke-subnet" = [ { - range_name = var.ip_range_pods + range_name = "random-ip-range-pods" ip_cidr_range = "10.1.0.0/16" }, { - range_name = var.ip_range_services + range_name = "random-ip-range-services" ip_cidr_range = "10.2.0.0/20" }, ] } diff --git a/examples/regional_private_node_pool_oauth_scopes/outputs.tf b/examples/regional_private_node_pool_oauth_scopes/outputs.tf index 464ed303c3..8f600dad7e 100644 --- a/examples/regional_private_node_pool_oauth_scopes/outputs.tf +++ b/examples/regional_private_node_pool_oauth_scopes/outputs.tf @@ -16,102 +16,112 @@ output "cluster_name" { description = "Cluster name" - value = "${module.gke.name}" + value = "module.gke.name" } output "type" { description = "Cluster type (regional / zonal)" - value = "${module.gke.type}" + value = "module.gke.type" } output "location" { description = "Cluster location (region if regional cluster, zone if zonal cluster)" - value = "${module.gke.location}" + value = "module.gke.location" } output "region" { description = "Cluster region" - value = "${module.gke.region}" + value = "module.gke.region" } output "zones" { description = "List of zones in which the cluster resides" - value = "${module.gke.zones}" + value = "module.gke.zones" } output "endpoint" { sensitive = true description = "Cluster endpoint" - value = "${module.gke.endpoint}" + value = "module.gke.endpoint" } output "min_master_version" { description = "Minimum master kubernetes version" - value = "${module.gke.min_master_version}" + value = "module.gke.min_master_version" } output "logging_service" { description = "Logging service used" - value = "${module.gke.logging_service}" + value = "module.gke.logging_service" } output "monitoring_service" { description = "Monitoring service used" - value = "${module.gke.monitoring_service}" + value = "module.gke.monitoring_service" } output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = "${module.gke.master_authorized_networks_config}" + value = "module.gke.master_authorized_networks_config" } output "master_version" { description = "Current master kubernetes version" - value = "${module.gke.master_version}" + value = "module.gke.master_version" } output "ca_certificate" { sensitive = true description = "Cluster ca certificate (base64 encoded)" - value = "${module.gke.ca_certificate}" + value = "module.gke.ca_certificate" } output "network_policy_enabled" { description = "Whether network policy enabled" - value = "${module.gke.network_policy_enabled}" + value = "module.gke.network_policy_enabled" } output "http_load_balancing_enabled" { description = "Whether http load balancing enabled" - value = "${module.gke.http_load_balancing_enabled}" + value = "module.gke.http_load_balancing_enabled" } output "horizontal_pod_autoscaling_enabled" { description = "Whether horizontal pod autoscaling enabled" - value = "${module.gke.horizontal_pod_autoscaling_enabled}" + value = "module.gke.horizontal_pod_autoscaling_enabled" } output "kubernetes_dashboard_enabled" { description = "Whether kubernetes dashboard enabled" - value = "${module.gke.kubernetes_dashboard_enabled}" + value = "module.gke.kubernetes_dashboard_enabled" } output "node_pools_names" { description = "List of node pools names" - value = "${module.gke.node_pools_names}" + value = "module.gke.node_pools_names" } output "node_pools_versions" { description = "List of node pools versions" - value = "${module.gke.node_pools_versions}" + value = "module.gke.node_pools_versions" } output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." - value = "${module.gke.service_account}" + value = "module.gke.service_account" } output "network_module" { description = "network module output" value = module.gke-network } + +output "subnets_ips" { + description = "The IP and cidrs of the subnets being created" + value = module.gke-network.subnets_ips +} + +output "subnets_secondary_ranges" { + description = "The secondary ranges associated with these subnets" + value = module.gke-network.subnets_secondary_ranges +} diff --git a/examples/regional_private_node_pool_oauth_scopes/provider.tf b/examples/regional_private_node_pool_oauth_scopes/provider.tf index 4317f93e11..d052ca7cdb 100644 --- a/examples/regional_private_node_pool_oauth_scopes/provider.tf +++ b/examples/regional_private_node_pool_oauth_scopes/provider.tf @@ -16,12 +16,8 @@ provider "google" { version = "2.18.0" - project = var.project_id - region = var.region } provider "google-beta" { version = "2.18.0" - project = var.project_id - region = var.region } diff --git a/examples/regional_private_node_pool_oauth_scopes/variables.tf b/examples/regional_private_node_pool_oauth_scopes/variables.tf index 07352a1a0b..03eedccd1f 100644 --- a/examples/regional_private_node_pool_oauth_scopes/variables.tf +++ b/examples/regional_private_node_pool_oauth_scopes/variables.tf @@ -14,30 +14,6 @@ * limitations under the License. */ -variable "cluster_name" { - description = "Name of the cluster" -} - variable "project_id" { description = "The project ID to host the cluster in" } - -variable "region" { - description = "The region to host the cluster in" -} - -variable "ip_range_pods" { - description = "The secondary ip range to use for pods" -} - -variable "ip_range_services" { - description = "The secondary ip range to use for pods" -} - -variable "network" { - description = "The VPC network name to host the cluster in" -} - -variable "subnet" { - description = "The subnetwork name to host the cluster in" -} From 9b2fa142f7a10bcde3681dae228660aeb07e20ee Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Tue, 19 Nov 2019 20:10:30 -0600 Subject: [PATCH 034/110] add .tmpl ext to autogen tf templates --- autogen/{auth.tf => auth.tf.tmpl} | 0 autogen/{cluster.tf => cluster.tf.tmpl} | 0 autogen/{dns.tf => dns.tf.tmpl} | 0 autogen/{main.tf => main.tf.tmpl} | 0 autogen/{masq.tf => masq.tf.tmpl} | 0 autogen/{networks.tf => networks.tf.tmpl} | 0 autogen/{outputs.tf => outputs.tf.tmpl} | 0 autogen/{sa.tf => sa.tf.tmpl} | 0 autogen/{variables.tf => variables.tf.tmpl} | 0 autogen/{versions.tf => versions.tf.tmpl} | 0 helpers/generate_modules/README.md | 186 +++++++++++++++++++ helpers/generate_modules/generate_modules.py | 6 +- 12 files changed, 190 insertions(+), 2 deletions(-) rename autogen/{auth.tf => auth.tf.tmpl} (100%) rename autogen/{cluster.tf => cluster.tf.tmpl} (100%) rename autogen/{dns.tf => dns.tf.tmpl} (100%) rename autogen/{main.tf => main.tf.tmpl} (100%) rename autogen/{masq.tf => masq.tf.tmpl} (100%) rename autogen/{networks.tf => networks.tf.tmpl} (100%) rename autogen/{outputs.tf => outputs.tf.tmpl} (100%) rename autogen/{sa.tf => sa.tf.tmpl} (100%) rename autogen/{variables.tf => variables.tf.tmpl} (100%) rename autogen/{versions.tf => versions.tf.tmpl} (100%) create mode 100644 helpers/generate_modules/README.md diff --git a/autogen/auth.tf b/autogen/auth.tf.tmpl similarity index 100% rename from autogen/auth.tf rename to autogen/auth.tf.tmpl diff --git a/autogen/cluster.tf b/autogen/cluster.tf.tmpl similarity index 100% rename from autogen/cluster.tf rename to autogen/cluster.tf.tmpl diff --git a/autogen/dns.tf b/autogen/dns.tf.tmpl similarity index 100% rename from autogen/dns.tf rename to autogen/dns.tf.tmpl diff --git a/autogen/main.tf b/autogen/main.tf.tmpl similarity index 100% rename from autogen/main.tf rename to autogen/main.tf.tmpl diff --git a/autogen/masq.tf b/autogen/masq.tf.tmpl similarity index 100% rename from autogen/masq.tf rename to autogen/masq.tf.tmpl diff --git a/autogen/networks.tf b/autogen/networks.tf.tmpl similarity index 100% rename from autogen/networks.tf rename to autogen/networks.tf.tmpl diff --git a/autogen/outputs.tf b/autogen/outputs.tf.tmpl similarity index 100% rename from autogen/outputs.tf rename to autogen/outputs.tf.tmpl diff --git a/autogen/sa.tf b/autogen/sa.tf.tmpl similarity index 100% rename from autogen/sa.tf rename to autogen/sa.tf.tmpl diff --git a/autogen/variables.tf b/autogen/variables.tf.tmpl similarity index 100% rename from autogen/variables.tf rename to autogen/variables.tf.tmpl diff --git a/autogen/versions.tf b/autogen/versions.tf.tmpl similarity index 100% rename from autogen/versions.tf rename to autogen/versions.tf.tmpl diff --git a/helpers/generate_modules/README.md b/helpers/generate_modules/README.md new file mode 100644 index 0000000000..66623aa0c4 --- /dev/null +++ b/helpers/generate_modules/README.md @@ -0,0 +1,186 @@ +# Terraform Kubernetes Engine Module + +This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. +The resources/services/activations/deletions that this module will create/trigger are: +- Create a GKE cluster with the provided addons +- Create GKE Node Pool(s) with provided configuration and attach to cluster +- Replace the default kube-dns configmap if `stub_domains` are provided +- Activate network policy if `network_policy` is true +- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `configure_ip_masq` is true + +Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. + + +## Compatibility + +This module is meant for use with Terraform 0.12. If you haven't +[upgraded][terraform-0.12-upgrade] and need a Terraform +0.11.x-compatible version of this module, the last released version +intended for Terraform 0.11.x is [3.0.0]. + +## Usage +There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: + +```hcl +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google" + project_id = "" + name = "gke-test-1" + region = "us-central1" + zones = ["us-central1-a", "us-central1-b", "us-central1-f"] + network = "vpc-01" + subnetwork = "us-central1-01" + ip_range_pods = "us-central1-01-gke-01-pods" + ip_range_services = "us-central1-01-gke-01-services" + http_load_balancing = false + horizontal_pod_autoscaling = true + kubernetes_dashboard = true + network_policy = true + + node_pools = [ + { + name = "default-node-pool" + machine_type = "n1-standard-2" + min_count = 1 + max_count = 100 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + }, + ] + + node_pools_oauth_scopes = { + all = [] + + default-node-pool = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + + node_pools_labels = { + all = {} + + default-node-pool = { + default-node-pool = true + } + } + + node_pools_metadata = { + all = {} + + default-node-pool = { + node-pool-metadata-custom-value = "my-node-pool" + } + } + + node_pools_taints = { + all = [] + + default-node-pool = [ + { + key = "default-node-pool" + value = true + effect = "PREFER_NO_SCHEDULE" + }, + ] + } + + node_pools_tags = { + all = [] + + default-node-pool = [ + "default-node-pool", + ] + } +} +``` + + +Then perform the following commands on the root folder: + +- `terraform init` to get the plugins +- `terraform plan` to see the infrastructure plan +- `terraform apply` to apply the infrastructure build +- `terraform destroy` to destroy the built infrastructure + +## Upgrade to v3.0.0 + +v3.0.0 is a breaking release. Refer to the +[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. + +## Upgrade to v2.0.0 + +v2.0.0 is a breaking release. Refer to the +[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. + +## Upgrade to v1.0.0 + +Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. + +In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. + + + + +## Requirements + +Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: + +1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. +2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). +3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. +4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. + +The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. + +### Software Dependencies +#### Kubectl +- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x +#### Terraform and Plugins +- [Terraform](https://www.terraform.io/downloads.html) 0.12 +- [Terraform Provider for GCP][terraform-provider-google] v2.9 + +### Configure a Service Account +In order to execute this module you must have a Service Account with the +following project roles: +- roles/compute.viewer +- roles/container.clusterAdmin +- roles/container.developer +- roles/iam.serviceAccountAdmin +- roles/iam.serviceAccountUser +- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) + +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + +### Enable APIs +In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: + +- Compute Engine API - compute.googleapis.com +- Kubernetes Engine API - container.googleapis.com + +## File structure +The project has the following folders and files: + +- /: root folder +- /examples: Examples for using this module and sub module. +- /helpers: Helper scripts. +- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file). +- /test: Folders with files for testing the module (see Testing section on this file). +- /main.tf: `main` file for the public module, contains all the resources to create. +- /variables.tf: Variables for the public cluster module. +- /output.tf: The outputs for the public cluster module. +- /README.MD: This file. +- /modules: Private and beta sub modules. + + +[upgrading-to-v2.0]: docs/upgrading_to_v2.0.md +[upgrading-to-v3.0]: docs/upgrading_to_v3.0.md +[terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google +[3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 +[terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index b98b8bb69e..f542ddf966 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -82,9 +82,11 @@ def main(argv): lstrip_blocks=True, ) templates = env.list_templates() - for template_file in templates: - for module in MODULES: + for module in MODULES: + for template_file in templates: template = env.get_template(template_file) + if template_file.endswith(".tf.tmpl"): + template_file=template_file.replace(".tf.tmpl",".tf") rendered = template.render( module.template_options(BASE_TEMPLATE_OPTIONS) ) From 7e9e119e91ebbae29f309dab713238785132f89d Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Tue, 19 Nov 2019 20:13:23 -0600 Subject: [PATCH 035/110] remove random file generated during testing --- helpers/generate_modules/README.md | 186 ----------------------------- 1 file changed, 186 deletions(-) delete mode 100644 helpers/generate_modules/README.md diff --git a/helpers/generate_modules/README.md b/helpers/generate_modules/README.md deleted file mode 100644 index 66623aa0c4..0000000000 --- a/helpers/generate_modules/README.md +++ /dev/null @@ -1,186 +0,0 @@ -# Terraform Kubernetes Engine Module - -This module handles opinionated Google Cloud Platform Kubernetes Engine cluster creation and configuration with Node Pools, IP MASQ, Network Policy, etc. -The resources/services/activations/deletions that this module will create/trigger are: -- Create a GKE cluster with the provided addons -- Create GKE Node Pool(s) with provided configuration and attach to cluster -- Replace the default kube-dns configmap if `stub_domains` are provided -- Activate network policy if `network_policy` is true -- Add `ip-masq-agent` configmap with provided `non_masquerade_cidrs` if `configure_ip_masq` is true - -Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. - - -## Compatibility - -This module is meant for use with Terraform 0.12. If you haven't -[upgraded][terraform-0.12-upgrade] and need a Terraform -0.11.x-compatible version of this module, the last released version -intended for Terraform 0.11.x is [3.0.0]. - -## Usage -There are multiple examples included in the [examples](./examples/) folder but simple usage is as follows: - -```hcl -module "gke" { - source = "terraform-google-modules/kubernetes-engine/google" - project_id = "" - name = "gke-test-1" - region = "us-central1" - zones = ["us-central1-a", "us-central1-b", "us-central1-f"] - network = "vpc-01" - subnetwork = "us-central1-01" - ip_range_pods = "us-central1-01-gke-01-pods" - ip_range_services = "us-central1-01-gke-01-services" - http_load_balancing = false - horizontal_pod_autoscaling = true - kubernetes_dashboard = true - network_policy = true - - node_pools = [ - { - name = "default-node-pool" - machine_type = "n1-standard-2" - min_count = 1 - max_count = 100 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - }, - ] - - node_pools_oauth_scopes = { - all = [] - - default-node-pool = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } - - node_pools_labels = { - all = {} - - default-node-pool = { - default-node-pool = true - } - } - - node_pools_metadata = { - all = {} - - default-node-pool = { - node-pool-metadata-custom-value = "my-node-pool" - } - } - - node_pools_taints = { - all = [] - - default-node-pool = [ - { - key = "default-node-pool" - value = true - effect = "PREFER_NO_SCHEDULE" - }, - ] - } - - node_pools_tags = { - all = [] - - default-node-pool = [ - "default-node-pool", - ] - } -} -``` - - -Then perform the following commands on the root folder: - -- `terraform init` to get the plugins -- `terraform plan` to see the infrastructure plan -- `terraform apply` to apply the infrastructure build -- `terraform destroy` to destroy the built infrastructure - -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - - - - -## Requirements - -Before this module can be used on a project, you must ensure that the following pre-requisites are fulfilled: - -1. Terraform and kubectl are [installed](#software-dependencies) on the machine where Terraform is executed. -2. The Service Account you execute the module with has the right [permissions](#configure-a-service-account). -3. The Compute Engine and Kubernetes Engine APIs are [active](#enable-apis) on the project you will launch the cluster in. -4. If you are using a Shared VPC, the APIs must also be activated on the Shared VPC host project and your service account needs the proper permissions there. - -The [project factory](https://github.com/terraform-google-modules/terraform-google-project-factory) can be used to provision projects with the correct APIs active and the necessary Shared VPC connections. - -### Software Dependencies -#### Kubectl -- [kubectl](https://github.com/kubernetes/kubernetes/releases) 1.9.x -#### Terraform and Plugins -- [Terraform](https://www.terraform.io/downloads.html) 0.12 -- [Terraform Provider for GCP][terraform-provider-google] v2.9 - -### Configure a Service Account -In order to execute this module you must have a Service Account with the -following project roles: -- roles/compute.viewer -- roles/container.clusterAdmin -- roles/container.developer -- roles/iam.serviceAccountAdmin -- roles/iam.serviceAccountUser -- roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) - -Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: -- roles/resourcemanager.projectIamAdmin - -### Enable APIs -In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: - -- Compute Engine API - compute.googleapis.com -- Kubernetes Engine API - container.googleapis.com - -## File structure -The project has the following folders and files: - -- /: root folder -- /examples: Examples for using this module and sub module. -- /helpers: Helper scripts. -- /scripts: Scripts for specific tasks on module (see Infrastructure section on this file). -- /test: Folders with files for testing the module (see Testing section on this file). -- /main.tf: `main` file for the public module, contains all the resources to create. -- /variables.tf: Variables for the public cluster module. -- /output.tf: The outputs for the public cluster module. -- /README.MD: This file. -- /modules: Private and beta sub modules. - - -[upgrading-to-v2.0]: docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: docs/upgrading_to_v3.0.md -[terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google -[3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 -[terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html From 85c3e3fe69768e743f81d2db78d81aeca03f2e8d Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Tue, 19 Nov 2019 20:18:17 -0600 Subject: [PATCH 036/110] pylint --- helpers/generate_modules/generate_modules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/generate_modules/generate_modules.py b/helpers/generate_modules/generate_modules.py index f542ddf966..a61329caf8 100755 --- a/helpers/generate_modules/generate_modules.py +++ b/helpers/generate_modules/generate_modules.py @@ -86,7 +86,7 @@ def main(argv): for template_file in templates: template = env.get_template(template_file) if template_file.endswith(".tf.tmpl"): - template_file=template_file.replace(".tf.tmpl",".tf") + template_file = template_file.replace(".tf.tmpl", ".tf") rendered = template.render( module.template_options(BASE_TEMPLATE_OPTIONS) ) From 03ed750950705142161e4e5f349ab93f87c4f85a Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Wed, 20 Nov 2019 14:38:35 +0800 Subject: [PATCH 037/110] Removed kubernetes dashboard add config. The kubernetes addon has been removed by GKE --- modules/private-cluster/cluster.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index c8051255bf..249baa28f6 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -79,10 +79,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } From c7ec3a9cec3a41d7019b8c2721f72b4f487a2b06 Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Wed, 20 Nov 2019 15:10:24 +0800 Subject: [PATCH 038/110] Update outputs.tf --- modules/private-cluster/outputs.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index dea7b5c7b5..54080bfa21 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names From 465b7c7f85d6237b2872db1c25052f9f3e2ed3ef Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Wed, 20 Nov 2019 15:11:44 +0800 Subject: [PATCH 039/110] Update main.tf --- modules/private-cluster/main.tf | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index f0d307311c..01af2436ef 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -80,8 +80,7 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled - + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) @@ -105,7 +104,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** From 591b9a93478bbd8928e0149b1c499ebc84bb8cb6 Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Wed, 20 Nov 2019 16:37:46 +0800 Subject: [PATCH 040/110] Updated autogen to remove the inclusion of kubernetes dashboard addon. --- autogen/cluster.tf | 4 ---- autogen/main.tf | 2 -- autogen/outputs.tf | 5 ----- 3 files changed, 11 deletions(-) diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 4791220591..1b8a585cd4 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -119,10 +119,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/autogen/main.tf b/autogen/main.tf index 841444ea44..3316d8eb75 100644 --- a/autogen/main.tf +++ b/autogen/main.tf @@ -105,7 +105,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled {% if beta_cluster %} # BETA features @@ -139,7 +138,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled {% if beta_cluster %} # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled diff --git a/autogen/outputs.tf b/autogen/outputs.tf index 842502ecea..44222bee6e 100644 --- a/autogen/outputs.tf +++ b/autogen/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names From 034d7e87748f3c21bbacb8bf91b3d34b3277a0a3 Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Wed, 20 Nov 2019 16:39:29 +0800 Subject: [PATCH 041/110] Removed kubernets dashboard addon from all modules. --- modules/beta-private-cluster-update-variant/cluster.tf | 4 ---- modules/beta-private-cluster-update-variant/main.tf | 2 -- modules/beta-private-cluster-update-variant/outputs.tf | 5 ----- modules/beta-private-cluster/cluster.tf | 4 ---- modules/beta-private-cluster/main.tf | 2 -- modules/beta-private-cluster/outputs.tf | 5 ----- modules/beta-public-cluster/cluster.tf | 4 ---- modules/beta-public-cluster/main.tf | 2 -- modules/beta-public-cluster/outputs.tf | 5 ----- modules/private-cluster-update-variant/cluster.tf | 4 ---- modules/private-cluster-update-variant/main.tf | 2 -- modules/private-cluster-update-variant/outputs.tf | 5 ----- 12 files changed, 44 deletions(-) diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 10c206f35c..32bdbf1527 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -111,10 +111,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 5b235ce00f..1632296df3 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -93,7 +93,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false @@ -125,7 +124,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index fb3f29c401..947c806067 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c363dacf9f..b6f91a38d8 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -111,10 +111,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 5b235ce00f..1632296df3 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -93,7 +93,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false @@ -125,7 +124,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index fb3f29c401..947c806067 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 1f5eee84aa..9c966c3534 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -111,10 +111,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9668b6f1ea..0dcce31340 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -93,7 +93,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled # BETA features cluster_output_istio_disabled = google_container_cluster.primary.addons_config.0.istio_config != null && length(google_container_cluster.primary.addons_config.0.istio_config) == 1 ? google_container_cluster.primary.addons_config.0.istio_config.0.disabled : false @@ -125,7 +124,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled cluster_cloudrun_enabled = var.cloudrun diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index fb3f29c401..947c806067 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index d7fc2dd736..967c02337b 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -79,10 +79,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index f0d307311c..7826dfff18 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -80,7 +80,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) @@ -105,7 +104,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index dea7b5c7b5..54080bfa21 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names From 02f24dd56ad1fc963845730f672608f09b73cbfc Mon Sep 17 00:00:00 2001 From: Jan Toebes Date: Wed, 20 Nov 2019 12:37:38 +0100 Subject: [PATCH 042/110] Add dependency to 'wait for resource' Add dependency so helm is not going to access the cluster before it is up and running --- autogen/outputs.tf.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/outputs.tf.tmpl b/autogen/outputs.tf.tmpl index 842502ecea..22fe89862b 100644 --- a/autogen/outputs.tf.tmpl +++ b/autogen/outputs.tf.tmpl @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } From 8e6df72d92693944d9a1c2f441b65c600ba6aead Mon Sep 17 00:00:00 2001 From: Jan Toebes Date: Wed, 20 Nov 2019 12:40:41 +0100 Subject: [PATCH 043/110] Add dependency to 'wait for resource' --- modules/beta-private-cluster-update-variant/outputs.tf | 1 + modules/beta-private-cluster/outputs.tf | 1 + modules/beta-public-cluster/outputs.tf | 1 + modules/private-cluster-update-variant/outputs.tf | 1 + modules/private-cluster/outputs.tf | 1 + outputs.tf | 1 + 6 files changed, 6 insertions(+) diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index fb3f29c401..e4a98aa447 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index fb3f29c401..e4a98aa447 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index fb3f29c401..e4a98aa447 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index dea7b5c7b5..3d6e9c8dab 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index dea7b5c7b5..3d6e9c8dab 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } diff --git a/outputs.tf b/outputs.tf index dea7b5c7b5..3d6e9c8dab 100644 --- a/outputs.tf +++ b/outputs.tf @@ -54,6 +54,7 @@ output "endpoint" { */ google_container_cluster.primary, google_container_node_pool.pools, + null_resource.wait_for_cluster.id, ] } From 868eecddd7cf56fff5ac0ea354c0ecfaa3328b0d Mon Sep 17 00:00:00 2001 From: Nicolas Bazire Date: Tue, 27 Aug 2019 11:24:22 +0200 Subject: [PATCH 044/110] Add support for local_ssd_count in node config. --- CHANGELOG.md | 2 ++ README.md | 1 + autogen/README.md | 1 + autogen/cluster.tf.tmpl | 6 ++++-- cluster.tf | 6 ++++-- examples/node_pool/main.tf | 1 + modules/beta-private-cluster-update-variant/README.md | 1 + modules/beta-private-cluster-update-variant/cluster.tf | 6 ++++-- modules/beta-private-cluster/README.md | 1 + modules/beta-private-cluster/cluster.tf | 6 ++++-- modules/beta-public-cluster/README.md | 1 + modules/beta-public-cluster/cluster.tf | 6 ++++-- modules/private-cluster-update-variant/README.md | 1 + modules/private-cluster-update-variant/cluster.tf | 6 ++++-- modules/private-cluster/README.md | 1 + modules/private-cluster/cluster.tf | 6 ++++-- 16 files changed, 38 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e738732c..9b9d87223a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for setting node_locations on node pools. [#303] * Fix for specifying `node_count` on node pools when autoscaling is disabled. [#311] * Added submodule for installing Anthos Config Management. [#268] +* Support for `local_ssd_count` in node pool configuration. [#244] ## [v5.1.1] - 2019-10-25 @@ -245,6 +246,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238 [#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241 [#250]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/250 +[#244]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/244 [#236]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/236 [#217]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/217 [#234]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/234 diff --git a/README.md b/README.md index 15f6aff13b..1068fc7f30 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ module "gke" { machine_type = "n1-standard-2" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/autogen/README.md b/autogen/README.md index 3efe785ff0..4165c9b8d8 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -60,6 +60,7 @@ module "gke" { {% endif %} min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index 4791220591..27448db32d 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -384,8 +384,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/cluster.tf b/cluster.tf index 072a60fb14..07801f226a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -185,8 +185,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 5bc0f53407..1ae41a53bb 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -50,6 +50,7 @@ module "gke" { machine_type = "n1-standard-2" min_count = 1 max_count = 2 + local_ssd_count = 0 disk_size_gb = 30 disk_type = "pd-standard" accelerator_count = 1 diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d2b5197726..51efdb5467 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -51,6 +51,7 @@ module "gke" { node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 10c206f35c..8cc2311055 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -352,8 +352,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 96ad9abf5a..71e281d32e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -51,6 +51,7 @@ module "gke" { node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c363dacf9f..95b78225e7 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -280,8 +280,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fcf78cccc..5052b306a9 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -46,6 +46,7 @@ module "gke" { node_locations = "us-central1-b,us-central1-c" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 1f5eee84aa..638d65af9d 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -275,8 +275,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index fa9cdb8852..f73d070cb1 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -48,6 +48,7 @@ module "gke" { machine_type = "n1-standard-2" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index d7fc2dd736..af422abb2d 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -262,8 +262,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 5465544b82..d23f84b68b 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -48,6 +48,7 @@ module "gke" { machine_type = "n1-standard-2" min_count = 1 max_count = 100 + local_ssd_count = 0 disk_size_gb = 100 disk_type = "pd-standard" image_type = "COS" diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index c8051255bf..4567530763 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -190,8 +190,10 @@ resource "google_container_node_pool" "pools" { var.node_pools_tags[var.node_pools[count.index]["name"]], ) - disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) - disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) + disk_size_gb = lookup(var.node_pools[count.index], "disk_size_gb", 100) + disk_type = lookup(var.node_pools[count.index], "disk_type", "pd-standard") + service_account = lookup( var.node_pools[count.index], "service_account", From 3c7f472c5cc5ec8f7f5a5bade2e5153f8fe1b518 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 20 Nov 2019 12:23:14 +0000 Subject: [PATCH 045/110] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b9d87223a..85f598f641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ Extending the adopted spec, each change should have a link to its corresponding * Fix for specifying `node_count` on node pools when autoscaling is disabled. [#311] * Added submodule for installing Anthos Config Management. [#268] * Support for `local_ssd_count` in node pool configuration. [#244] +* Wait for cluster to be ready before returning endpoint. [#340] ## [v5.1.1] - 2019-10-25 @@ -229,6 +230,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 [#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268 [#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303 From bf48602efa65752374e1774c0df503f238e24ab3 Mon Sep 17 00:00:00 2001 From: Pavel Zhukov Date: Wed, 13 Nov 2019 18:40:56 +0200 Subject: [PATCH 046/110] Make node_pools_* params optional and allow pool-specific overrides. [test/deploy_service] Add retry logic to wait until the nginx service becomes reachable [build] Add 120 seconds delay after the 'prepare' step --- Makefile | 2 +- autogen/cluster.tf.tmpl | 61 +++++++-------- autogen/main.tf.tmpl | 3 +- autogen/variables.tf.tmpl | 25 ++++--- autogen/variables_defaults.tf | 74 +++++++++++++++++++ build/int.cloudbuild.yaml | 4 +- build/lint.cloudbuild.yaml | 2 +- cluster.tf | 33 +++++---- examples/node_pool/main.tf | 16 ---- .../cluster.tf | 63 ++++++++-------- .../main.tf | 1 + .../variables.tf | 7 ++ .../variables_defaults.tf | 72 ++++++++++++++++++ modules/beta-private-cluster/cluster.tf | 37 +++++----- modules/beta-private-cluster/main.tf | 1 + modules/beta-private-cluster/variables.tf | 7 ++ .../variables_defaults.tf | 72 ++++++++++++++++++ modules/beta-public-cluster/cluster.tf | 37 +++++----- modules/beta-public-cluster/main.tf | 1 + modules/beta-public-cluster/variables.tf | 7 ++ .../beta-public-cluster/variables_defaults.tf | 72 ++++++++++++++++++ .../private-cluster-update-variant/cluster.tf | 59 ++++++++------- .../variables.tf | 4 + .../variables_defaults.tf | 62 ++++++++++++++++ modules/private-cluster/cluster.tf | 33 +++++---- modules/private-cluster/variables.tf | 4 + modules/private-cluster/variables_defaults.tf | 62 ++++++++++++++++ .../deploy_service/controls/kubectl.rb | 6 ++ variables.tf | 4 + variables_defaults.tf | 62 ++++++++++++++++ 30 files changed, 712 insertions(+), 181 deletions(-) create mode 100644 autogen/variables_defaults.tf create mode 100644 modules/beta-private-cluster-update-variant/variables_defaults.tf create mode 100644 modules/beta-private-cluster/variables_defaults.tf create mode 100644 modules/beta-public-cluster/variables_defaults.tf create mode 100644 modules/private-cluster-update-variant/variables_defaults.tf create mode 100644 modules/private-cluster/variables_defaults.tf create mode 100644 variables_defaults.tf diff --git a/Makefile b/Makefile index 736cad34ce..09dc66079c 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.6 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index 27448db32d..77b3b5b84e 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -252,10 +252,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -264,10 +264,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -276,8 +276,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -286,8 +286,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -314,7 +314,9 @@ resource "google_container_node_pool" "pools" { // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null {% endif %} - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -350,16 +352,16 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, @@ -367,8 +369,8 @@ resource "google_container_node_pool" "pools" { {% if beta_cluster %} dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -378,10 +380,10 @@ resource "google_container_node_pool" "pools" { } {% endif %} tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -396,8 +398,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -431,6 +433,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + {% if update_variant %} create_before_destroy = true {% endif %} diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 841444ea44..15d6d3983d 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -147,7 +147,8 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled - cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ + + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] # /BETA features diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index ad5cc44e34..e58f2ef487 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -79,7 +79,7 @@ variable "node_version" { } variable "master_authorized_networks_config" { - type = list(object({cidr_blocks = list(object({cidr_block = string, display_name = string}))})) + type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -181,9 +183,10 @@ variable "node_pools_metadata" { {% if beta_cluster %} variable "node_pools_taints" { - type = map(list(object({key=string,value=string,effect=string}))) + type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -195,6 +198,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -205,6 +209,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -356,10 +361,11 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." - type = list(object({state = string, key_name = string})) - default = [{ - state = "DECRYPTED" - key_name = "" + type = list(object({ state = string, key_name = string })) + + default = [{ + state = "DECRYPTED" + key_name = "" }] } @@ -375,7 +381,8 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." - default = [{ + + default = [{ "enabled" = false }] } @@ -429,8 +436,8 @@ variable "release_channel" { } variable "enable_shielded_nodes" { - type = bool + type = bool description = "Enable Shielded Nodes features on all nodes in this cluster" - default = false + default = false } {% endif %} diff --git a/autogen/variables_defaults.tf b/autogen/variables_defaults.tf new file mode 100644 index 0000000000..ccc9b0eed3 --- /dev/null +++ b/autogen/variables_defaults.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{{ autogeneration_note }} + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + +{% if beta_cluster %} + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + +{% endif %} + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index e78d0eb2ba..af0560641f 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -19,7 +19,7 @@ steps: args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && download_acm'] - id: prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && prepare_environment && sleep 120'] env: - 'TF_VAR_org_id=$_ORG_ID' - 'TF_VAR_folder_id=$_FOLDER_ID' @@ -309,6 +309,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 02b9e5327b..82c43b9fa6 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -24,4 +24,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.5.4' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0' diff --git a/cluster.tf b/cluster.tf index 07801f226a..73d17fc26a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -131,7 +131,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -164,25 +166,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -197,8 +199,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -214,6 +216,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 1ae41a53bb..b2f1d010ed 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -72,20 +72,10 @@ module "gke" { }, ] - node_pools_oauth_scopes = { - all = [] - pool-01 = [] - pool-02 = [] - pool-03 = [] - } - node_pools_metadata = { - all = {} pool-01 = { shutdown-script = file("${path.module}/data/shutdown-script.sh") } - pool-02 = {} - pool-03 = {} } node_pools_labels = { @@ -95,8 +85,6 @@ module "gke" { pool-01 = { pool-01-example = true } - pool-02 = {} - pool-03 = {} } node_pools_taints = { @@ -114,8 +102,6 @@ module "gke" { effect = "PREFER_NO_SCHEDULE" }, ] - pool-02 = [] - pool-03 = [] } node_pools_tags = { @@ -125,8 +111,6 @@ module "gke" { pool-01 = [ "pool-01-example", ] - pool-02 = [] - pool-03 = [] } } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 8cc2311055..93e7072f31 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -235,10 +235,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -247,10 +247,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -259,8 +259,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -269,8 +269,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -286,7 +286,9 @@ resource "google_container_node_pool" "pools" { location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -320,24 +322,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -346,10 +348,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -364,8 +366,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -396,7 +398,8 @@ resource "google_container_node_pool" "pools" { } lifecycle { - ignore_changes = [initial_node_count] + ignore_changes = [initial_node_count] + create_before_destroy = true } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 5b235ce00f..bc466df2b2 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d04ed5ac1f..65335a674c 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -352,6 +357,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -370,6 +376,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-private-cluster-update-variant/variables_defaults.tf b/modules/beta-private-cluster-update-variant/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-private-cluster-update-variant/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 95b78225e7..e5f4e4510d 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -214,7 +214,9 @@ resource "google_container_node_pool" "pools" { location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -248,24 +250,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -274,10 +276,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -292,8 +294,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -325,6 +327,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 5b235ce00f..bc466df2b2 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d04ed5ac1f..65335a674c 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -352,6 +357,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -370,6 +376,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-private-cluster/variables_defaults.tf b/modules/beta-private-cluster/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-private-cluster/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 638d65af9d..72b513cd57 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -209,7 +209,9 @@ resource "google_container_node_pool" "pools" { location = local.location // use node_locations if provided, defaults to cluster level node_locations if not specified node_locations = lookup(var.node_pools[count.index], "node_locations", "") != "" ? split(",", var.node_pools[count.index]["node_locations"]) : null - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -243,24 +245,24 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) dynamic "taint" { for_each = concat( - var.node_pools_taints["all"], - var.node_pools_taints[var.node_pools[count.index]["name"]], + local.node_pools_taints["all"], + local.node_pools_taints[var.node_pools[count.index]["name"]], ) content { effect = taint.value.effect @@ -269,10 +271,10 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -287,8 +289,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -320,6 +322,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9668b6f1ea..dbc346ca0c 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -132,6 +132,7 @@ locals { cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled cluster_vertical_pod_autoscaling_enabled = local.cluster_output_vertical_pod_autoscaling_enabled + cluster_workload_identity_config = var.identity_namespace == "" ? [] : [{ identity_namespace = var.identity_namespace }] diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a1057f1843..b875aea14f 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -203,6 +207,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] @@ -328,6 +333,7 @@ variable "default_max_pods_per_node" { variable "database_encryption" { description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." type = list(object({ state = string, key_name = string })) + default = [{ state = "DECRYPTED" key_name = "" @@ -346,6 +352,7 @@ variable "enable_binary_authorization" { variable "pod_security_policy_config" { description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ "enabled" = false }] diff --git a/modules/beta-public-cluster/variables_defaults.tf b/modules/beta-public-cluster/variables_defaults.tf new file mode 100644 index 0000000000..9a75764306 --- /dev/null +++ b/modules/beta-public-cluster/variables_defaults.tf @@ -0,0 +1,72 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_taints = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_taints + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index af422abb2d..eebb4cd2ab 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -159,10 +159,10 @@ resource "random_id" "name" { labels = join(",", sort( concat( - keys(var.node_pools_labels["all"]), - values(var.node_pools_labels["all"]), - keys(var.node_pools_labels[var.node_pools[count.index]["name"]]), - values(var.node_pools_labels[var.node_pools[count.index]["name"]]) + keys(local.node_pools_labels["all"]), + values(local.node_pools_labels["all"]), + keys(local.node_pools_labels[var.node_pools[count.index]["name"]]), + values(local.node_pools_labels[var.node_pools[count.index]["name"]]) ) ) ) @@ -171,10 +171,10 @@ resource "random_id" "name" { metadata = join(",", sort( concat( - keys(var.node_pools_metadata["all"]), - values(var.node_pools_metadata["all"]), - keys(var.node_pools_metadata[var.node_pools[count.index]["name"]]), - values(var.node_pools_metadata[var.node_pools[count.index]["name"]]) + keys(local.node_pools_metadata["all"]), + values(local.node_pools_metadata["all"]), + keys(local.node_pools_metadata[var.node_pools[count.index]["name"]]), + values(local.node_pools_metadata[var.node_pools[count.index]["name"]]) ) ) ) @@ -183,8 +183,8 @@ resource "random_id" "name" { oauth_scopes = join(",", sort( concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]] ) ) ) @@ -193,8 +193,8 @@ resource "random_id" "name" { tags = join(",", sort( concat( - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]] + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]] ) ) ) @@ -208,7 +208,9 @@ resource "google_container_node_pool" "pools" { name = random_id.name.*.hex[count.index] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -241,25 +243,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -274,8 +276,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -290,7 +292,8 @@ resource "google_container_node_pool" "pools" { } lifecycle { - ignore_changes = [initial_node_count] + ignore_changes = [initial_node_count] + create_before_destroy = true } diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 508a4f1b96..52ad93f973 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/modules/private-cluster-update-variant/variables_defaults.tf b/modules/private-cluster-update-variant/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/modules/private-cluster-update-variant/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 4567530763..94f3637919 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -136,7 +136,9 @@ resource "google_container_node_pool" "pools" { name = var.node_pools[count.index]["name"] project = var.project_id location = local.location - cluster = google_container_cluster.primary.name + + cluster = google_container_cluster.primary.name + version = lookup(var.node_pools[count.index], "auto_upgrade", false) ? "" : lookup( var.node_pools[count.index], "version", @@ -169,25 +171,25 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_labels["all"], - var.node_pools_labels[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_labels["all"], + local.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, - lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, - var.node_pools_metadata["all"], - var.node_pools_metadata[var.node_pools[count.index]["name"]], + lookup(lookup(local.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(local.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, + local.node_pools_metadata["all"], + local.node_pools_metadata[var.node_pools[count.index]["name"]], { "disable-legacy-endpoints" = var.disable_legacy_metadata_endpoints }, ) tags = concat( - lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], - lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], - var.node_pools_tags["all"], - var.node_pools_tags[var.node_pools[count.index]["name"]], + lookup(local.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(local.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], + local.node_pools_tags["all"], + local.node_pools_tags[var.node_pools[count.index]["name"]], ) local_ssd_count = lookup(var.node_pools[count.index], "local_ssd_count", 0) @@ -202,8 +204,8 @@ resource "google_container_node_pool" "pools" { preemptible = lookup(var.node_pools[count.index], "preemptible", false) oauth_scopes = concat( - var.node_pools_oauth_scopes["all"], - var.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], + local.node_pools_oauth_scopes["all"], + local.node_pools_oauth_scopes[var.node_pools[count.index]["name"]], ) guest_accelerator = [ @@ -219,6 +221,7 @@ resource "google_container_node_pool" "pools" { lifecycle { ignore_changes = [initial_node_count] + } timeouts { diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 508a4f1b96..52ad93f973 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/modules/private-cluster/variables_defaults.tf b/modules/private-cluster/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/modules/private-cluster/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} diff --git a/test/integration/deploy_service/controls/kubectl.rb b/test/integration/deploy_service/controls/kubectl.rb index 2d4a473d2c..67921b1298 100644 --- a/test/integration/deploy_service/controls/kubectl.rb +++ b/test/integration/deploy_service/controls/kubectl.rb @@ -58,6 +58,12 @@ it "is reachable" do expect { + 10.times do + unless host(service_load_balancer_ip, port: 8080, protocol: 'tcp').reachable? + puts "Nginx is not reachable, retrying.." + sleep 10 + end + end RestClient.get(service_load_balancer_address) }.to_not raise_exception end diff --git a/variables.tf b/variables.tf index 58cf1f4685..5b89c692ec 100644 --- a/variables.tf +++ b/variables.tf @@ -163,6 +163,7 @@ variable "node_pools_labels" { type = map(map(string)) description = "Map of maps containing node labels by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -173,6 +174,7 @@ variable "node_pools_metadata" { type = map(map(string)) description = "Map of maps containing node metadata by node-pool name" + # Default is being set in variables_defaults.tf default = { all = {} default-node-pool = {} @@ -183,6 +185,7 @@ variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" + # Default is being set in variables_defaults.tf default = { all = [] default-node-pool = [] @@ -193,6 +196,7 @@ variable "node_pools_oauth_scopes" { type = map(list(string)) description = "Map of lists containing node oauth scopes by node-pool name" + # Default is being set in variables_defaults.tf default = { all = ["https://www.googleapis.com/auth/cloud-platform"] default-node-pool = [] diff --git a/variables_defaults.tf b/variables_defaults.tf new file mode 100644 index 0000000000..145685f074 --- /dev/null +++ b/variables_defaults.tf @@ -0,0 +1,62 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file was automatically generated from a template in ./autogen + +# Setup dynamic default values for variables which can't be setup using +# the standard terraform "variable default" functionality + +locals { + node_pools_labels = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_labels + ) + + node_pools_metadata = merge( + { all = {} }, + { default-node-pool = {} }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : {}] + ), + var.node_pools_metadata + ) + + node_pools_tags = merge( + { all = [] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_tags + ) + + node_pools_oauth_scopes = merge( + { all = ["https://www.googleapis.com/auth/cloud-platform"] }, + { default-node-pool = [] }, + zipmap( + [for node_pool in var.node_pools : node_pool["name"]], + [for node_pool in var.node_pools : []] + ), + var.node_pools_oauth_scopes + ) +} From 2d650a190a61e8f54eefb5c5e372e31a47c692b3 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Wed, 20 Nov 2019 14:05:02 -0600 Subject: [PATCH 047/110] Changes to the variables --- autogen/README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index e61fe093c3..82a24dd35d 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -190,21 +190,30 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | +| accelerator_count | The number of the guest accelerator cards exposed to this instance | 0 | Optional | +| accelerator_type | The accelerator type resource to expose to the instance | " " | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| effect | Effect for the taint | | Required | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| key | The key required for the taint | | Required | | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | +| name | The name of the node pool | | Required | +| node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1 | | Required (when autoscaling is false) | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| node_metadata | Options to expose the node metadata to the workload running on the node | | Required | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| sandbox_type | Sandbox to use for pods in the node pool | | Required | | service_account | The service account to be used by the Node VMs | " " | Optional | +| tags | The list of instance tags applied to all nodes | | Required | +| value | The value for the taint | | Required | ## File structure The project has the following folders and files: From 5fca1690e1089e3ffa7c1f7c8e0dae1344d28948 Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Thu, 21 Nov 2019 12:26:00 +0800 Subject: [PATCH 048/110] Fixed the files and updated README. --- README.md | 1 - cluster.tf | 4 ---- main.tf | 2 -- modules/beta-private-cluster-update-variant/README.md | 1 - modules/beta-private-cluster/README.md | 1 - modules/beta-public-cluster/README.md | 1 - modules/private-cluster-update-variant/README.md | 1 - modules/private-cluster/README.md | 1 - modules/private-cluster/main.tf | 2 +- outputs.tf | 5 ----- 10 files changed, 1 insertion(+), 18 deletions(-) diff --git a/README.md b/README.md index 15f6aff13b..514563ca17 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/cluster.tf b/cluster.tf index 072a60fb14..d518027963 100644 --- a/cluster.tf +++ b/cluster.tf @@ -79,10 +79,6 @@ resource "google_container_cluster" "primary" { disabled = ! var.horizontal_pod_autoscaling } - kubernetes_dashboard { - disabled = ! var.kubernetes_dashboard - } - network_policy_config { disabled = ! var.network_policy } diff --git a/main.tf b/main.tf index 1090227fd8..754fcefe04 100644 --- a/main.tf +++ b/main.tf @@ -80,7 +80,6 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - cluster_output_kubernetes_dashboard_enabled = google_container_cluster.primary.addons_config.0.kubernetes_dashboard.0.disabled cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) @@ -105,7 +104,6 @@ locals { cluster_network_policy_enabled = ! local.cluster_output_network_policy_enabled cluster_http_load_balancing_enabled = ! local.cluster_output_http_load_balancing_enabled cluster_horizontal_pod_autoscaling_enabled = ! local.cluster_output_horizontal_pod_autoscaling_enabled - cluster_kubernetes_dashboard_enabled = ! local.cluster_output_kubernetes_dashboard_enabled } /****************************************** diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index d2b5197726..cc36e4dcd5 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -216,7 +216,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | identity\_namespace | Workload Identity namespace | | intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 96ad9abf5a..54fcb702a2 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -216,7 +216,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | identity\_namespace | Workload Identity namespace | | intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fcf78cccc..94d38c0946 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -207,7 +207,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | identity\_namespace | Workload Identity namespace | | intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index fa9cdb8852..c40e260a59 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -193,7 +193,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 5465544b82..926c61a461 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -193,7 +193,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 01af2436ef..7826dfff18 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -80,7 +80,7 @@ locals { cluster_output_network_policy_enabled = google_container_cluster.primary.addons_config.0.network_policy_config.0.disabled cluster_output_http_load_balancing_enabled = google_container_cluster.primary.addons_config.0.http_load_balancing.0.disabled cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled - + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/outputs.tf b/outputs.tf index dea7b5c7b5..54080bfa21 100644 --- a/outputs.tf +++ b/outputs.tf @@ -103,11 +103,6 @@ output "horizontal_pod_autoscaling_enabled" { value = local.cluster_horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = local.cluster_kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = local.cluster_node_pools_names From 6fc8854cffbb98bb9aac87db49dbe6405d422aba Mon Sep 17 00:00:00 2001 From: Shashindran Vijayan Date: Thu, 21 Nov 2019 13:37:30 +0800 Subject: [PATCH 049/110] Updated README and variables.tf --- README.md | 2 -- autogen/README.md | 1 - autogen/variables.tf.tmpl | 6 ------ modules/beta-private-cluster-update-variant/README.md | 2 -- modules/beta-private-cluster-update-variant/variables.tf | 6 ------ modules/beta-private-cluster/README.md | 2 -- modules/beta-private-cluster/variables.tf | 6 ------ modules/beta-public-cluster/README.md | 2 -- modules/beta-public-cluster/variables.tf | 6 ------ modules/private-cluster-update-variant/README.md | 2 -- modules/private-cluster-update-variant/variables.tf | 6 ------ modules/private-cluster/README.md | 2 -- modules/private-cluster/variables.tf | 6 ------ variables.tf | 6 ------ 14 files changed, 55 deletions(-) diff --git a/README.md b/README.md index c7e0b264b4..60931a99af 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true node_pools = [ @@ -147,7 +146,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/autogen/README.md b/autogen/README.md index 4165c9b8d8..c8e956a76e 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -39,7 +39,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true {% if private_cluster %} enable_private_endpoint = true diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index ad5cc44e34..8d6c3e5985 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 4490f9cee8..2f52021bd7 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -36,7 +36,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true enable_private_endpoint = true enable_private_nodes = true @@ -168,7 +167,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | | istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d04ed5ac1f..eb7c78f502 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 37aa5b52b0..a60877eca3 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -36,7 +36,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true enable_private_endpoint = true enable_private_nodes = true @@ -168,7 +167,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | | istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d04ed5ac1f..eb7c78f502 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 3dd5b263d7..00da4429f2 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -34,7 +34,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true istio = true cloudrun = true @@ -160,7 +159,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | | istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a1057f1843..e0c1d89db9 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 26ccb3b08f..37f95ccf7d 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -36,7 +36,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true enable_private_endpoint = true enable_private_nodes = true @@ -155,7 +154,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 508a4f1b96..7c82afc753 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index b3f16a1497..adbb3b8042 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -36,7 +36,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" http_load_balancing = false horizontal_pod_autoscaling = true - kubernetes_dashboard = true network_policy = true enable_private_endpoint = true enable_private_nodes = true @@ -155,7 +154,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 508a4f1b96..7c82afc753 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" diff --git a/variables.tf b/variables.tf index 58cf1f4685..d50dd0150d 100644 --- a/variables.tf +++ b/variables.tf @@ -96,12 +96,6 @@ variable "http_load_balancing" { default = true } -variable "kubernetes_dashboard" { - type = bool - description = "Enable kubernetes dashboard addon" - default = false -} - variable "network_policy" { type = bool description = "Enable network policy addon" From 1e21211a6f38c21d7746d9779993d88299a96b7b Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Fri, 15 Nov 2019 18:08:43 +0200 Subject: [PATCH 050/110] Added cluster autoscaling Updated docs Added tests for cluster autoscaling in node_pool fixture * Fix #93 --- autogen/cluster.tf.tmpl | 13 ++++++++++ autogen/main.tf.tmpl | 14 +++++++++++ autogen/variables.tf.tmpl | 14 ++++++++++- cluster.tf | 1 + examples/node_pool/README.md | 1 + examples/node_pool/main.tf | 1 + examples/node_pool/variables.tf | 11 +++++++++ .../README.md | 1 + .../cluster.tf | 13 ++++++++++ .../main.tf | 14 +++++++++++ .../variables.tf | 12 ++++++++++ modules/beta-private-cluster/README.md | 1 + modules/beta-private-cluster/cluster.tf | 13 ++++++++++ modules/beta-private-cluster/main.tf | 14 +++++++++++ modules/beta-private-cluster/variables.tf | 12 ++++++++++ modules/beta-public-cluster/README.md | 1 + modules/beta-public-cluster/cluster.tf | 13 ++++++++++ modules/beta-public-cluster/main.tf | 14 +++++++++++ modules/beta-public-cluster/variables.tf | 12 ++++++++++ .../private-cluster-update-variant/cluster.tf | 1 + .../variables.tf | 1 - modules/private-cluster/cluster.tf | 1 + modules/private-cluster/variables.tf | 1 - test/fixtures/node_pool/example.tf | 10 ++++++++ test/integration/node_pool/controls/gcloud.rb | 24 +++++++++++++++++++ variables.tf | 1 - 26 files changed, 210 insertions(+), 4 deletions(-) diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index 27448db32d..e0f8ae6bea 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -45,6 +45,7 @@ resource "google_container_cluster" "primary" { } } + {% if beta_cluster %} dynamic "release_channel" { for_each = local.release_channel @@ -62,6 +63,18 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service {% if beta_cluster %} + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 841444ea44..47aa3d9be8 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -50,6 +50,20 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal {% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) {% endif %} diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index ad5cc44e34..9ffa2efc97 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -178,8 +178,20 @@ variable "node_pools_metadata" { default-node-pool = {} } } - {% if beta_cluster %} + +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({key=string,value=string,effect=string}))) description = "Map of lists containing node taints by node-pool name" diff --git a/cluster.tf b/cluster.tf index 07801f226a..cbacb8ec1e 100644 --- a/cluster.tf +++ b/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 237b3f0b6f..7b8867b9c8 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -7,6 +7,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 1ae41a53bb..6388164817 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -36,6 +36,7 @@ module "gke" { create_service_account = false remove_default_node_pool = true disable_legacy_metadata_endpoints = false + cluster_autoscaling = var.cluster_autoscaling node_pools = [ { diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 040c78d2c4..82c02e5756 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -52,3 +52,14 @@ variable "compute_engine_service_account" { description = "Service account to associate to the nodes in the cluster" } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 51efdb5467..673c92cd35 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -142,6 +142,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 8cc2311055..dc41132a18 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 5b235ce00f..1f0b3ca8f3 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d04ed5ac1f..8a592fa7bf 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 71e281d32e..440662c8d0 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -142,6 +142,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 95b78225e7..0aa5ae49bd 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 5b235ce00f..1f0b3ca8f3 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d04ed5ac1f..8a592fa7bf 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 5052b306a9..fc2bd51554 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -137,6 +137,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 638d65af9d..2e66119985 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -41,6 +41,7 @@ resource "google_container_cluster" "primary" { } } + dynamic "release_channel" { for_each = local.release_channel @@ -55,6 +56,18 @@ resource "google_container_cluster" "primary" { logging_service = var.logging_service monitoring_service = var.monitoring_service + cluster_autoscaling { + enabled = var.cluster_autoscaling.enabled + dynamic "resource_limits" { + for_each = local.autoscalling_resource_limits + content { + resource_type = lookup(resource_limits.value, "resource_type") + minimum = lookup(resource_limits.value, "minimum") + maximum = lookup(resource_limits.value, "maximum") + } + } + } + enable_binary_authorization = var.enable_binary_authorization enable_intranode_visibility = var.enable_intranode_visibility default_max_pods_per_node = var.default_max_pods_per_node diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 9668b6f1ea..c44a3ad315 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -45,6 +45,20 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] + limits = var.cluster_autoscaling.resource_limits + + autoscalling_resource_limits = concat( + var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + resource_type = "cpu" + minimum = local.limits["min_cpu_cores"] + maximum = local.limits["max_cpu_cores"] + }] : [], + var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + resource_type = "memory" + minimum = local.limits["min_memory_gb"] + maximum = local.limits["max_memory_gb"] + }] : [] + ) custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index a1057f1843..41d70b7a52 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -179,6 +179,18 @@ variable "node_pools_metadata" { } } +variable "cluster_autoscaling" { + type = object({ + enabled = bool + resource_limits = map(number) + }) + default = { + enabled = false + resource_limits = {} + } + description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" +} + variable "node_pools_taints" { type = map(list(object({ key = string, value = string, effect = string }))) description = "Map of lists containing node taints by node-pool name" diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index af422abb2d..2c5ada707c 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 508a4f1b96..3398d01422 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 4567530763..de23a99b4a 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -42,6 +42,7 @@ resource "google_container_cluster" "primary" { } + subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 508a4f1b96..3398d01422 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 28d51f2357..2df8b89df2 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -26,5 +26,15 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account + + cluster_autoscaling = { + enabled = true + resource_limits = { + max_cpu_cores = 20 + min_cpu_cores = 5 + max_memory_gb = 30 + min_memory_gb = 10 + } + } } diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 69a15e8293..c08b61f1c6 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -33,6 +33,30 @@ end end + describe "cluster-autoscaling" do + it "has the expected cluster autoscaling settings" do + expect(data['autoscaling']).to eq({ + "autoprovisioningNodePoolDefaults" => { + "oauthScopes" => %w(https://www.googleapis.com/auth/logging.write https://www.googleapis.com/auth/monitoring), + "serviceAccount" => "default" + }, + "enableNodeAutoprovisioning" => true, + "resourceLimits" => [ + { + "maximum" => "20", + "minimum" => "5", + "resourceType" => "cpu" + }, + { + "maximum" => "30", + "minimum" => "10", + "resourceType" => "memory" + } + ] + }) + end + end + describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } diff --git a/variables.tf b/variables.tf index 58cf1f4685..bd110cbf42 100644 --- a/variables.tf +++ b/variables.tf @@ -178,7 +178,6 @@ variable "node_pools_metadata" { default-node-pool = {} } } - variable "node_pools_tags" { type = map(list(string)) description = "Map of lists containing node network tags by node-pool name" From 598ea5be45911bbc5cccb754ce16c749f3a56c06 Mon Sep 17 00:00:00 2001 From: Alekhya Lakkadi Date: Thu, 21 Nov 2019 12:04:03 -0800 Subject: [PATCH 051/110] formatted outputs.tf file --- .../outputs.tf | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/regional_private_node_pool_oauth_scopes/outputs.tf b/examples/regional_private_node_pool_oauth_scopes/outputs.tf index 8f600dad7e..13105641e9 100644 --- a/examples/regional_private_node_pool_oauth_scopes/outputs.tf +++ b/examples/regional_private_node_pool_oauth_scopes/outputs.tf @@ -16,99 +16,99 @@ output "cluster_name" { description = "Cluster name" - value = "module.gke.name" + value = module.gke.name } output "type" { description = "Cluster type (regional / zonal)" - value = "module.gke.type" + value = module.gke.type } output "location" { description = "Cluster location (region if regional cluster, zone if zonal cluster)" - value = "module.gke.location" + value = module.gke.location } output "region" { description = "Cluster region" - value = "module.gke.region" + value = module.gke.region } output "zones" { description = "List of zones in which the cluster resides" - value = "module.gke.zones" + value = module.gke.zones } output "endpoint" { sensitive = true description = "Cluster endpoint" - value = "module.gke.endpoint" + value = module.gke.endpoint } output "min_master_version" { description = "Minimum master kubernetes version" - value = "module.gke.min_master_version" + value = module.gke.min_master_version } output "logging_service" { description = "Logging service used" - value = "module.gke.logging_service" + value = module.gke.logging_service } output "monitoring_service" { description = "Monitoring service used" - value = "module.gke.monitoring_service" + value = module.gke.monitoring_service } output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = "module.gke.master_authorized_networks_config" + value = module.gke.master_authorized_networks_config } output "master_version" { description = "Current master kubernetes version" - value = "module.gke.master_version" + value = module.gke.master_version } output "ca_certificate" { sensitive = true description = "Cluster ca certificate (base64 encoded)" - value = "module.gke.ca_certificate" + value = module.gke.ca_certificate } output "network_policy_enabled" { description = "Whether network policy enabled" - value = "module.gke.network_policy_enabled" + value = module.gke.network_policy_enabled } output "http_load_balancing_enabled" { description = "Whether http load balancing enabled" - value = "module.gke.http_load_balancing_enabled" + value = module.gke.http_load_balancing_enabled } output "horizontal_pod_autoscaling_enabled" { description = "Whether horizontal pod autoscaling enabled" - value = "module.gke.horizontal_pod_autoscaling_enabled" + value = module.gke.horizontal_pod_autoscaling_enabled } output "kubernetes_dashboard_enabled" { description = "Whether kubernetes dashboard enabled" - value = "module.gke.kubernetes_dashboard_enabled" + value = module.gke.kubernetes_dashboard_enabled } output "node_pools_names" { description = "List of node pools names" - value = "module.gke.node_pools_names" + value = module.gke.node_pools_names } output "node_pools_versions" { description = "List of node pools versions" - value = "module.gke.node_pools_versions" + value = module.gke.node_pools_versions } output "service_account" { description = "The service account to default running nodes as if not overridden in `node_pools`." - value = "module.gke.service_account" + value = module.gke.service_account } output "network_module" { From 63c8b6f3237accdc8f7ee6f3e4ed51108dba9427 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 21 Nov 2019 22:43:16 +0000 Subject: [PATCH 052/110] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85f598f641..a849f89183 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for `local_ssd_count` in node pool configuration. [#244] * Wait for cluster to be ready before returning endpoint. [#340] +### Removed + +* **Breaking**: Removed support for enabling the Kubernetes dashboard, as this is deprecated on GKE. [#337] + ## [v5.1.1] - 2019-10-25 ### Fixed @@ -230,6 +234,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#337]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/337 [#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 [#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268 [#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311 From d1098b5b31b2731e6a398d47f6824f6d6b36008f Mon Sep 17 00:00:00 2001 From: Bohdan Yurov Date: Wed, 6 Nov 2019 15:24:07 +0200 Subject: [PATCH 053/110] Fixes #180: Add tests for beta submodules/examples https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/180 Added tests for the beta private cluster. --- .kitchen.yml | 13 ++ build/int.cloudbuild.yaml | 20 ++ examples/simple_regional_beta/README.md | 5 + examples/simple_regional_beta/main.tf | 38 ++-- examples/simple_regional_beta/variables.tf | 33 +++ test/ci/beta-cluster.yml | 18 ++ test/fixtures/beta_cluster/main.tf | 78 +++++++ test/fixtures/beta_cluster/network.tf | 44 ++++ test/fixtures/beta_cluster/outputs.tf | 84 ++++++++ test/fixtures/beta_cluster/variables.tf | 1 + .../beta_cluster/controls/gcloud.rb | 204 ++++++++++++++++++ test/integration/beta_cluster/controls/gcp.rb | 31 +++ test/integration/beta_cluster/inspec.yml | 33 +++ test/setup/iam.tf | 10 + test/setup/main.tf | 2 - 15 files changed, 595 insertions(+), 19 deletions(-) create mode 100644 test/ci/beta-cluster.yml create mode 100644 test/fixtures/beta_cluster/main.tf create mode 100644 test/fixtures/beta_cluster/network.tf create mode 100644 test/fixtures/beta_cluster/outputs.tf create mode 120000 test/fixtures/beta_cluster/variables.tf create mode 100644 test/integration/beta_cluster/controls/gcloud.rb create mode 100644 test/integration/beta_cluster/controls/gcp.rb create mode 100644 test/integration/beta_cluster/inspec.yml diff --git a/.kitchen.yml b/.kitchen.yml index 6e9a7a24d8..10033a8a91 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -155,6 +155,19 @@ suites: systems: - name: workload_metadata_config backend: local + - name: "beta_cluster" + driver: + root_module_directory: test/fixtures/beta_cluster + verifier: + systems: + - name: gcloud + backend: local + controls: + - gcloud + - name: gcp + backend: gcp + controls: + - gcp - name: "deploy_service" driver: root_module_directory: test/fixtures/deploy_service diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index e78d0eb2ba..6982d474c8 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -244,6 +244,26 @@ steps: - verify workload-metadata-config-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local'] +- id: create beta-cluster-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create beta-cluster-local'] +- id: converge beta-cluster-local + waitFor: + - create beta-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge beta-cluster-local'] +- id: verify beta-cluster-local + waitFor: + - converge beta-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify beta-cluster-local'] +#- id: destroy beta-cluster-local +# waitFor: +# - verify beta-cluster-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy beta-cluster-local'] - id: create deploy-service-local waitFor: - prepare diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index 72bb221d9f..32bfc8fbfd 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -10,17 +10,22 @@ This example illustrates how to create a simple cluster with beta features. | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index 0863cc51de..55acfaed79 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -24,23 +24,27 @@ provider "google-beta" { } module "gke" { - source = "../../modules/beta-public-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - istio = var.istio - cloudrun = var.cloudrun - node_metadata = var.node_metadata - sandbox_enabled = var.sandbox_enabled - remove_default_node_pool = var.remove_default_node_pool - node_pools = var.node_pools + source = "../../modules/beta-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = var.regional + region = var.region + zones = var.zones + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = var.compute_engine_service_account == "create" + service_account = var.compute_engine_service_account + istio = var.istio + cloudrun = var.cloudrun + node_metadata = var.node_metadata + sandbox_enabled = var.sandbox_enabled + remove_default_node_pool = var.remove_default_node_pool + node_pools = var.node_pools + database_encryption = var.database_encryption + enable_binary_authorization = var.enable_binary_authorization + pod_security_policy_config = var.pod_security_policy_config } data "google_client_config" "default" { diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index ed16642774..58e1ae7433 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -85,3 +85,36 @@ variable "node_pools" { }, ] } + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "pod_security_policy_config" { + description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ + "enabled" = false + }] +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} diff --git a/test/ci/beta-cluster.yml b/test/ci/beta-cluster.yml new file mode 100644 index 0000000000..dd4ce29302 --- /dev/null +++ b/test/ci/beta-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "beta-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf new file mode 100644 index 0000000000..4431b715f3 --- /dev/null +++ b/test/fixtures/beta_cluster/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +provider "google" { + version = "~> 2.18.0" + project = var.project_id + region = var.region +} + +provider "google-beta" { + version = "~> 2.18.0" + project = var.project_id + region = var.region +} + +locals { + name = "beta-cluster-${random_string.suffix.result}" +} + +resource "google_kms_key_ring" "db" { + location = var.region + name = "${local.name}-db" +} + +resource "google_kms_crypto_key" "db" { + name = local.name + key_ring = google_kms_key_ring.db.self_link +} + +module "this" { + source = "../../../examples/simple_regional_beta" + + cluster_name_suffix = "-${random_string.suffix.result}" + project_id = var.project_id + regional = false + region = var.region + zones = slice(var.zones, 0, 1) + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = "create" + + // Beta features + istio = true + + database_encryption = [{ + state = "ENCRYPTED" + key_name = google_kms_crypto_key.db.self_link + }] + + cloudrun = true + + enable_binary_authorization = true + + pod_security_policy_config = [{ + enabled = true + }] + + node_metadata = "EXPOSE" +} + +data "google_client_config" "default" { +} diff --git a/test/fixtures/beta_cluster/network.tf b/test/fixtures/beta_cluster/network.tf new file mode 100644 index 0000000000..0a3f091958 --- /dev/null +++ b/test/fixtures/beta_cluster/network.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/beta_cluster/outputs.tf b/test/fixtures/beta_cluster/outputs.tf new file mode 100644 index 0000000000..da60cf87cf --- /dev/null +++ b/test/fixtures/beta_cluster/outputs.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.this.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.this.cluster_name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.this.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.this.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.this.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.this.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.this.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.this.service_account +} + +output "database_encryption_key_name" { + value = google_kms_crypto_key.db.self_link +} diff --git a/test/fixtures/beta_cluster/variables.tf b/test/fixtures/beta_cluster/variables.tf new file mode 120000 index 0000000000..c28fc18c01 --- /dev/null +++ b/test/fixtures/beta_cluster/variables.tf @@ -0,0 +1 @@ +../deploy_service/variables.tf \ No newline at end of file diff --git a/test/integration/beta_cluster/controls/gcloud.rb b/test/integration/beta_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..455a81cb61 --- /dev/null +++ b/test/integration/beta_cluster/controls/gcloud.rb @@ -0,0 +1,204 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +service_account = attribute('service_account') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + "istioConfig" => {}, + "cloudRunConfig" => {}, + }) + end + + it "has the expected binaryAuthorization config" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "has the expected nodeMetadata conseal config" do + expect(data['nodeConfig']['workloadMetadataConfig']).to eq({ + "nodeMetadata" => 'EXPOSE', + }) + end + + it "has the expected podSecurityPolicyConfig config" do + expect(data['podSecurityPolicyConfig']).to eq({ + "enabled" => true, + }) + end + + it "has the expected databaseEncryption config" do + expect(data['databaseEncryption']).to eq({ + "state" => 'ENCRYPTED', + "keyName" => attribute('database_encryption_key_name'), + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "uses an automatically created service account" do + expect(node_pools).to include( + including( + "config" => including( + "serviceAccount" => service_account, + ), + ), + ) + end + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/beta_cluster/controls/gcp.rb b/test/integration/beta_cluster/controls/gcp.rb new file mode 100644 index 0000000000..6e9ade64ff --- /dev/null +++ b/test/integration/beta_cluster/controls/gcp.rb @@ -0,0 +1,31 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gcp" do + title "Native InSpec Resources" + + service_account = attribute("service_account") + project_id = attribute("project_id") + + if service_account.start_with? "projects/" + service_account_name = service_account + else + service_account_name = "projects/#{project_id}/serviceAccounts/#{service_account}" + end + + describe google_service_account name: service_account_name do + its("display_name") { should eq "Terraform-managed service account for cluster #{attribute("cluster_name")}" } + its("project_id") { should eq project_id } + end +end diff --git a/test/integration/beta_cluster/inspec.yml b/test/integration/beta_cluster/inspec.yml new file mode 100644 index 0000000000..66062ea35d --- /dev/null +++ b/test/integration/beta_cluster/inspec.yml @@ -0,0 +1,33 @@ +name: beta_cluster +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: service_account + required: true + type: string + - name: service_account + required: true + type: string + - name: database_encryption_key_name + required: true + type: string diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 7ff4de74bc..a06a490c24 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -16,6 +16,7 @@ locals { int_required_roles = [ + "roles/cloudkms.admin", "roles/cloudkms.cryptoKeyEncrypterDecrypter", "roles/compute.networkAdmin", "roles/container.admin", @@ -58,3 +59,12 @@ resource "google_project_iam_member" "int_test" { resource "google_service_account_key" "int_test" { service_account_id = google_service_account.int_test.id } + +resource "google_project_iam_binding" "kubernetes_engine_kms_access" { + project = module.gke-project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + + members = [ + "serviceAccount:service-${module.gke-project.project_number}@container-engine-robot.iam.gserviceaccount.com", + ] +} diff --git a/test/setup/main.tf b/test/setup/main.tf index 70e10c46a3..f974c7408e 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -24,8 +24,6 @@ module "gke-project" { folder_id = var.folder_id billing_account = var.billing_account - auto_create_network = true - activate_apis = [ "bigquery-json.googleapis.com", "cloudkms.googleapis.com", From 1f2edb63d59bc257809b92fad6b25bc4ec9880ae Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 22 Nov 2019 08:32:15 -0600 Subject: [PATCH 054/110] remove kubernetes_dashboard_enabled --- modules/safer-cluster/README.md | 1 - modules/safer-cluster/outputs.tf | 5 ----- 2 files changed, 6 deletions(-) diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 4b51d4d5e3..2d99d26f36 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -254,7 +254,6 @@ For simplicity, we suggest using `roles/container.admin` and | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/modules/safer-cluster/outputs.tf b/modules/safer-cluster/outputs.tf index bb4fb79667..1c688e27d5 100644 --- a/modules/safer-cluster/outputs.tf +++ b/modules/safer-cluster/outputs.tf @@ -102,11 +102,6 @@ output "horizontal_pod_autoscaling_enabled" { value = module.gke.horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = module.gke.kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = module.gke.node_pools_names From 4f3df803244c0fc14a824c616144fa3fdea00232 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 22 Nov 2019 10:20:38 -0500 Subject: [PATCH 055/110] Remove obsolete dashboard output --- examples/regional_private_node_pool_oauth_scopes/README.md | 1 - examples/regional_private_node_pool_oauth_scopes/outputs.tf | 5 ----- 2 files changed, 6 deletions(-) diff --git a/examples/regional_private_node_pool_oauth_scopes/README.md b/examples/regional_private_node_pool_oauth_scopes/README.md index 3ae6a8ca77..fa9f673894 100644 --- a/examples/regional_private_node_pool_oauth_scopes/README.md +++ b/examples/regional_private_node_pool_oauth_scopes/README.md @@ -18,7 +18,6 @@ This example illustrates how to create a private cluster with node pool specific | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | diff --git a/examples/regional_private_node_pool_oauth_scopes/outputs.tf b/examples/regional_private_node_pool_oauth_scopes/outputs.tf index 13105641e9..2df5357298 100644 --- a/examples/regional_private_node_pool_oauth_scopes/outputs.tf +++ b/examples/regional_private_node_pool_oauth_scopes/outputs.tf @@ -91,11 +91,6 @@ output "horizontal_pod_autoscaling_enabled" { value = module.gke.horizontal_pod_autoscaling_enabled } -output "kubernetes_dashboard_enabled" { - description = "Whether kubernetes dashboard enabled" - value = module.gke.kubernetes_dashboard_enabled -} - output "node_pools_names" { description = "List of node pools names" value = module.gke.node_pools_names From 509f6badc4abe5c11b02a300352b47ca3fed4b79 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Fri, 22 Nov 2019 09:23:10 -0600 Subject: [PATCH 056/110] fix int test --- build/int.cloudbuild.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index af0560641f..45e23b6193 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -64,6 +64,26 @@ steps: - verify shared-vpc-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy shared-vpc-local'] +- id: create safer-cluster-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create safer-cluster-local'] +- id: converge safer-cluster-local + waitFor: + - create safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge safer-cluster-local'] +- id: verify safer-cluster-local + waitFor: + - converge safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify safer-cluster-local'] +- id: destroy safer-cluster-local + waitFor: + - verify safer-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy safer-cluster-local'] - id: create simple-regional-local waitFor: - prepare From 932bd0e464370a0b772291bad1e45dcd6aa051d2 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Sat, 23 Nov 2019 12:30:33 -0600 Subject: [PATCH 057/110] hardcode vars --- examples/safer_cluster/README.md | 20 +-------- examples/safer_cluster/main.tf | 27 ++++++++---- examples/safer_cluster/network.tf | 16 +++---- examples/safer_cluster/outputs.tf | 24 ++++++++++ examples/safer_cluster/test_outputs.tf | 1 - examples/safer_cluster/variables.tf | 61 -------------------------- 6 files changed, 53 insertions(+), 96 deletions(-) delete mode 120000 examples/safer_cluster/test_outputs.tf diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 845b263d13..2e925ac85b 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -7,20 +7,9 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | Name | Description | Type | Default | Required | |------|-------------|:----:|:-----:|:-----:| -| cloudrun | Boolean to enable / disable CloudRun | bool | `"true"` | no | -| cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | -| ip\_range\_pods | The secondary ip range to use for pods | string | `"ip-range-pods"` | no | -| ip\_range\_services | The secondary ip range to use for pods | string | `"ip-range-scv"` | no | -| istio | Boolean to enable / disable Istio | bool | `"true"` | no | -| master\_auth\_subnetwork | The subnetwork that has access to cluster master | string | `"master-auth-subnet"` | no | -| master\_auth\_subnetwork\_cidr | The cidr block for the subnetwork that has access to cluster master | string | `"10.60.0.0/17"` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | string | `"172.16.0.0/28"` | no | -| network | The VPC network to host the cluster in | string | `"gke-network"` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | `"us-central1"` | no | -| subnetwork | The subnetwork to host the cluster in | string | `"gke-subnet"` | no | -| subnetwork\_cidr | The cidr block for the subnetwork to host the cluster in | string | `"10.0.0.0/17"` | no | ## Outputs @@ -29,18 +18,13 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | ca\_certificate | The cluster ca certificate (base64 encoded) | | client\_token | The bearer token for auth | | cluster\_name | Cluster name | -| ip\_range\_pods | The secondary IP range used for pods | -| ip\_range\_services | The secondary IP range used for services | | kubernetes\_endpoint | The cluster endpoint | | location | | -| master\_kubernetes\_version | The master Kubernetes version | -| network | | +| master\_kubernetes\_version | Kubernetes version of the master | | network\_name | The name of the VPC being created | -| project\_id | | -| region | | +| region | The region in which the cluster resides | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | -| subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index bcae52aac9..0aeca18009 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -14,8 +14,19 @@ * limitations under the License. */ +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + locals { - cluster_type = "safer-cluster" + cluster_type = "safer-cluster" + network_name = "safer-cluster-network-${random_string.suffix.result}" + subnet_name = "safer-cluster-subnet-${random_string.suffix.result}" + master_auth_subnetwork = "safer-cluster-master-subnet-${random_string.suffix.result}" + pods_range_name = "ip-range-pods-${random_string.suffix.result}" + svc_range_name = "ip-range-svc-${random_string.suffix.result}" } provider "google" { @@ -29,27 +40,27 @@ provider "google-beta" { module "gke" { source = "../../modules/safer-cluster/" project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + name = "${local.cluster_type}-cluster-${random_string.suffix.result}" regional = true region = var.region network = module.gcp-network.network_name subnetwork = module.gcp-network.subnets_names[0] - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services + ip_range_pods = local.pods_range_name + ip_range_services = local.svc_range_name compute_engine_service_account = var.compute_engine_service_account - master_ipv4_cidr_block = var.master_ipv4_cidr_block + master_ipv4_cidr_block = "172.16.0.0/28" master_authorized_networks_config = [ { cidr_blocks = [ { - cidr_block = var.master_auth_subnetwork_cidr + cidr_block = "10.60.0.0/17" display_name = "VPC" }, ] }, ] - istio = var.istio - cloudrun = var.cloudrun + istio = true + cloudrun = true } data "google_client_config" "default" { diff --git a/examples/safer_cluster/network.tf b/examples/safer_cluster/network.tf index 9bc38a9a3d..7ce9cf3946 100644 --- a/examples/safer_cluster/network.tf +++ b/examples/safer_cluster/network.tf @@ -18,29 +18,29 @@ module "gcp-network" { source = "terraform-google-modules/network/google" version = "~> 1.4.0" project_id = var.project_id - network_name = var.network + network_name = local.network_name subnets = [ { - subnet_name = var.subnetwork - subnet_ip = var.subnetwork_cidr + subnet_name = local.subnet_name + subnet_ip = "10.0.0.0/17" subnet_region = var.region }, { - subnet_name = var.master_auth_subnetwork - subnet_ip = var.master_auth_subnetwork_cidr + subnet_name = local.master_auth_subnetwork + subnet_ip = "10.60.0.0/17" subnet_region = var.region }, ] secondary_ranges = { - "${var.subnetwork}" = [ + "${local.subnet_name}" = [ { - range_name = var.ip_range_pods + range_name = local.pods_range_name ip_cidr_range = "192.168.0.0/18" }, { - range_name = var.ip_range_services + range_name = local.svc_range_name ip_cidr_range = "192.168.64.0/18" }, ] diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index eee2f4fbf2..72e09c0467 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -20,6 +20,20 @@ output "kubernetes_endpoint" { value = module.gke.endpoint } +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "location" { + value = module.gke.location +} + +output "master_kubernetes_version" { + description = "Kubernetes version of the master" + value = module.gke.master_version +} + output "client_token" { description = "The bearer token for auth" sensitive = true @@ -45,3 +59,13 @@ output "subnet_names" { description = "The names of the subnet being created" value = module.gcp-network.subnets_names } + +output "region" { + description = "The region in which the cluster resides" + value = module.gke.region +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} diff --git a/examples/safer_cluster/test_outputs.tf b/examples/safer_cluster/test_outputs.tf deleted file mode 120000 index 17b34213ba..0000000000 --- a/examples/safer_cluster/test_outputs.tf +++ /dev/null @@ -1 +0,0 @@ -../../test/fixtures/all_examples/test_outputs.tf \ No newline at end of file diff --git a/examples/safer_cluster/variables.tf b/examples/safer_cluster/variables.tf index 6b726747ca..dc3c20889a 100644 --- a/examples/safer_cluster/variables.tf +++ b/examples/safer_cluster/variables.tf @@ -19,73 +19,12 @@ variable "project_id" { description = "The project ID to host the cluster in" } -variable "cluster_name_suffix" { - type = string - description = "A suffix to append to the default cluster name" - default = "" -} - variable "region" { type = string description = "The region to host the cluster in" default = "us-central1" } -variable "network" { - type = string - description = "The VPC network to host the cluster in" - default = "gke-network" -} - -variable "subnetwork" { - type = string - description = "The subnetwork to host the cluster in" - default = "gke-subnet" -} -variable "subnetwork_cidr" { - type = string - description = "The cidr block for the subnetwork to host the cluster in" - default = "10.0.0.0/17" -} -variable "master_auth_subnetwork_cidr" { - type = string - description = "The cidr block for the subnetwork that has access to cluster master" - default = "10.60.0.0/17" -} -variable "master_auth_subnetwork" { - type = string - description = "The subnetwork that has access to cluster master" - default = "master-auth-subnet" -} -variable "ip_range_pods" { - type = string - description = "The secondary ip range to use for pods" - default = "ip-range-pods" -} - -variable "ip_range_services" { - type = string - description = "The secondary ip range to use for pods" - default = "ip-range-scv" -} -variable "istio" { - type = bool - description = "Boolean to enable / disable Istio" - default = true -} - -variable "cloudrun" { - type = bool - description = "Boolean to enable / disable CloudRun" - default = true -} - -variable "master_ipv4_cidr_block" { - type = string - description = "The IP range in CIDR notation to use for the hosted master network" - default = "172.16.0.0/28" -} - variable "compute_engine_service_account" { type = string description = "Service account to associate to the nodes in the cluster" From ee1593f7cbc38093a24bf605a73748b107e4e522 Mon Sep 17 00:00:00 2001 From: Rory Bramwell Date: Sun, 24 Nov 2019 09:52:11 -0500 Subject: [PATCH 058/110] Updated "upstream_nameservers" variable type to list(string) --- autogen/variables.tf.tmpl | 2 +- modules/beta-private-cluster-update-variant/variables.tf | 2 +- modules/beta-private-cluster/variables.tf | 2 +- modules/beta-public-cluster/variables.tf | 2 +- modules/private-cluster-update-variant/variables.tf | 2 +- modules/private-cluster/variables.tf | 2 +- variables.tf | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 71f2c65413..5b87f08e05 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -217,7 +217,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 890598553e..0f581e56cc 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -215,7 +215,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 890598553e..0f581e56cc 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -215,7 +215,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 717cd8d3c7..e27a7d0414 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -215,7 +215,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index f4fbe9fdf1..82c268aecf 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -204,7 +204,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index f4fbe9fdf1..82c268aecf 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -204,7 +204,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } diff --git a/variables.tf b/variables.tf index 03dc33b814..b9fdf45738 100644 --- a/variables.tf +++ b/variables.tf @@ -204,7 +204,7 @@ variable "stub_domains" { } variable "upstream_nameservers" { - type = "list" + type = list(string) description = "If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf" default = [] } From 98d36979e5f3a20b3a6f3a5e6f40d5fd736e9159 Mon Sep 17 00:00:00 2001 From: Rory Bramwell Date: Sun, 24 Nov 2019 10:05:20 -0500 Subject: [PATCH 059/110] Updated docs via autogen --- README.md | 2 +- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-public-cluster/README.md | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 60931a99af..4c15b0f311 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 2f52021bd7..22227f6a1e 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -200,7 +200,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index a60877eca3..c5c6b079ef 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -200,7 +200,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 00da4429f2..b0e2cfc85d 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -191,7 +191,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 37f95ccf7d..1786d3b46c 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -181,7 +181,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index adbb3b8042..0f5c3df935 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -181,7 +181,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs From 04fcb50db607dd5ae95324e6e44f48405fb3ff09 Mon Sep 17 00:00:00 2001 From: Andriy Kopachevskyy Date: Thu, 21 Nov 2019 15:05:29 +0200 Subject: [PATCH 060/110] Added cluster autoscaling - optimized input variable * Fix #93 --- autogen/cluster.tf.tmpl | 1 - autogen/main.tf.tmpl | 17 ++++++------- autogen/variables.tf.tmpl | 14 ++++++++--- cluster.tf | 1 - examples/node_pool/variables.tf | 14 ++++++++--- .../cluster.tf | 1 - .../main.tf | 25 ++++++++----------- .../variables.tf | 14 ++++++++--- modules/beta-private-cluster/cluster.tf | 1 - modules/beta-private-cluster/main.tf | 25 ++++++++----------- modules/beta-private-cluster/variables.tf | 14 ++++++++--- modules/beta-public-cluster/cluster.tf | 1 - modules/beta-public-cluster/main.tf | 25 ++++++++----------- modules/beta-public-cluster/variables.tf | 14 ++++++++--- .../private-cluster-update-variant/cluster.tf | 1 - modules/private-cluster/cluster.tf | 1 - test/fixtures/node_pool/example.tf | 12 ++++----- 17 files changed, 95 insertions(+), 86 deletions(-) diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index e0f8ae6bea..217d727c4a 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -45,7 +45,6 @@ resource "google_container_cluster" "primary" { } } - {% if beta_cluster %} dynamic "release_channel" { for_each = local.release_channel diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 47aa3d9be8..21fbb377bd 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -50,20 +50,17 @@ locals { node_version = var.regional ? local.node_version_regional : local.node_version_zonal {% if beta_cluster %} release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - limits = var.cluster_autoscaling.resource_limits - autoscalling_resource_limits = concat( - var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ resource_type = "cpu" - minimum = local.limits["min_cpu_cores"] - maximum = local.limits["max_cpu_cores"] - }] : [], - var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { resource_type = "memory" - minimum = local.limits["min_memory_gb"] - maximum = local.limits["max_memory_gb"] + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb }] : [] - ) + {% endif %} diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 9ffa2efc97..e3cde1481c 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -182,12 +182,18 @@ variable "node_pools_metadata" { variable "cluster_autoscaling" { type = object({ - enabled = bool - resource_limits = map(number) + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number }) default = { - enabled = false - resource_limits = {} + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/cluster.tf b/cluster.tf index cbacb8ec1e..07801f226a 100644 --- a/cluster.tf +++ b/cluster.tf @@ -42,7 +42,6 @@ resource "google_container_cluster" "primary" { } - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/examples/node_pool/variables.tf b/examples/node_pool/variables.tf index 82c02e5756..49570f241f 100644 --- a/examples/node_pool/variables.tf +++ b/examples/node_pool/variables.tf @@ -54,12 +54,18 @@ variable "compute_engine_service_account" { variable "cluster_autoscaling" { type = object({ - enabled = bool - resource_limits = map(number) + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number }) default = { - enabled = false - resource_limits = {} + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index dc41132a18..ff9998af03 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -41,7 +41,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "release_channel" { for_each = local.release_channel diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 1f0b3ca8f3..e1a4653479 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -45,20 +45,17 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - limits = var.cluster_autoscaling.resource_limits - - autoscalling_resource_limits = concat( - var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ - resource_type = "cpu" - minimum = local.limits["min_cpu_cores"] - maximum = local.limits["max_cpu_cores"] - }] : [], - var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ - resource_type = "memory" - minimum = local.limits["min_memory_gb"] - maximum = local.limits["max_memory_gb"] - }] : [] - ) + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 8a592fa7bf..86245ece24 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -181,12 +181,18 @@ variable "node_pools_metadata" { variable "cluster_autoscaling" { type = object({ - enabled = bool - resource_limits = map(number) + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number }) default = { - enabled = false - resource_limits = {} + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 0aa5ae49bd..f0986556b7 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -41,7 +41,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "release_channel" { for_each = local.release_channel diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 1f0b3ca8f3..e1a4653479 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -45,20 +45,17 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - limits = var.cluster_autoscaling.resource_limits - - autoscalling_resource_limits = concat( - var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ - resource_type = "cpu" - minimum = local.limits["min_cpu_cores"] - maximum = local.limits["max_cpu_cores"] - }] : [], - var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ - resource_type = "memory" - minimum = local.limits["min_memory_gb"] - maximum = local.limits["max_memory_gb"] - }] : [] - ) + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 8a592fa7bf..86245ece24 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -181,12 +181,18 @@ variable "node_pools_metadata" { variable "cluster_autoscaling" { type = object({ - enabled = bool - resource_limits = map(number) + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number }) default = { - enabled = false - resource_limits = {} + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 2e66119985..0eaabaace5 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -41,7 +41,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "release_channel" { for_each = local.release_channel diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index c44a3ad315..4272038e22 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -45,20 +45,17 @@ locals { master_version = var.regional ? local.master_version_regional : local.master_version_zonal node_version = var.regional ? local.node_version_regional : local.node_version_zonal release_channel = var.release_channel != null ? [{ channel : var.release_channel }] : [] - limits = var.cluster_autoscaling.resource_limits - - autoscalling_resource_limits = concat( - var.cluster_autoscaling.enabled && lookup(local.limits, "max_cpu_cores", 0) > lookup(local.limits, "min_cpu_cores", 0) ? [{ - resource_type = "cpu" - minimum = local.limits["min_cpu_cores"] - maximum = local.limits["max_cpu_cores"] - }] : [], - var.cluster_autoscaling.enabled && lookup(local.limits, "max_memory_gb", 0) > lookup(local.limits, "min_memory_gb", 0) ? [{ - resource_type = "memory" - minimum = local.limits["min_memory_gb"] - maximum = local.limits["max_memory_gb"] - }] : [] - ) + + autoscalling_resource_limits = var.cluster_autoscaling.enabled ? [{ + resource_type = "cpu" + minimum = var.cluster_autoscaling.min_cpu_cores + maximum = var.cluster_autoscaling.max_cpu_cores + }, { + resource_type = "memory" + minimum = var.cluster_autoscaling.min_memory_gb + maximum = var.cluster_autoscaling.max_memory_gb + }] : [] + custom_kube_dns_config = length(keys(var.stub_domains)) > 0 diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 41d70b7a52..3d55421b7c 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -181,12 +181,18 @@ variable "node_pools_metadata" { variable "cluster_autoscaling" { type = object({ - enabled = bool - resource_limits = map(number) + enabled = bool + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number }) default = { - enabled = false - resource_limits = {} + enabled = false + max_cpu_cores = 0 + min_cpu_cores = 0 + max_memory_gb = 0 + min_memory_gb = 0 } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 2c5ada707c..af422abb2d 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -42,7 +42,6 @@ resource "google_container_cluster" "primary" { } - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index de23a99b4a..4567530763 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -42,7 +42,6 @@ resource "google_container_cluster" "primary" { } - subnetwork = data.google_compute_subnetwork.gke_subnetwork.self_link min_master_version = local.master_version diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 2df8b89df2..94aa8e920d 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -28,13 +28,11 @@ module "example" { compute_engine_service_account = var.compute_engine_service_account cluster_autoscaling = { - enabled = true - resource_limits = { - max_cpu_cores = 20 - min_cpu_cores = 5 - max_memory_gb = 30 - min_memory_gb = 10 - } + enabled = true + max_cpu_cores = 20 + min_cpu_cores = 5 + max_memory_gb = 30 + min_memory_gb = 10 } } From b06c42541ca2fd830332635f58d3bb5818db77e3 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Mon, 25 Nov 2019 23:10:50 -0600 Subject: [PATCH 061/110] update int test to split between two projects --- examples/safer_cluster/README.md | 1 + examples/safer_cluster/outputs.tf | 5 ++++ test/fixtures/deploy_service/example.tf | 4 +-- test/fixtures/deploy_service/network.tf | 2 +- test/fixtures/disable_client_cert/example.tf | 6 ++-- test/fixtures/disable_client_cert/network.tf | 2 +- test/fixtures/node_pool/example.tf | 4 +-- test/fixtures/node_pool/network.tf | 2 +- .../node_pool_update_variant/example.tf | 4 +-- .../node_pool_update_variant/network.tf | 2 +- .../private_zonal_with_networking/example.tf | 2 +- .../private_zonal_with_networking/outputs.tf | 2 +- .../variables.tf | 3 +- test/fixtures/safer_cluster/example.tf | 4 +-- test/fixtures/safer_cluster/outputs.tf | 2 +- test/fixtures/sandbox_enabled/example.tf | 4 +-- test/fixtures/sandbox_enabled/network.tf | 2 +- test/fixtures/shared/outputs.tf | 2 +- test/fixtures/shared/variables.tf | 10 ++++--- test/fixtures/shared_vpc/example.tf | 6 ++-- test/fixtures/shared_vpc/network.tf | 2 +- test/fixtures/simple_regional/example.tf | 4 +-- test/fixtures/simple_regional/network.tf | 2 +- .../simple_regional_private/example.tf | 4 +-- .../simple_regional_private/network.tf | 4 +-- .../example.tf | 2 +- .../outputs.tf | 2 +- .../variables.tf | 5 ++-- test/fixtures/simple_zonal/example.tf | 2 +- test/fixtures/simple_zonal/network.tf | 2 +- test/fixtures/simple_zonal_private/example.tf | 4 +-- test/fixtures/simple_zonal_private/network.tf | 4 +-- test/fixtures/stub_domains/example.tf | 4 +-- test/fixtures/stub_domains/network.tf | 2 +- test/fixtures/stub_domains_private/main.tf | 8 ++--- .../example.tf | 4 +-- .../network.tf | 2 +- test/fixtures/upstream_nameservers/example.tf | 4 +-- test/fixtures/upstream_nameservers/network.tf | 2 +- .../workload_metadata_config/example.tf | 2 +- .../workload_metadata_config/network.tf | 6 ++-- test/setup/iam.tf | 28 ++++++++++++----- test/setup/main.tf | 30 ++++++++++++++++++- test/setup/outputs.tf | 8 ++--- 44 files changed, 129 insertions(+), 77 deletions(-) diff --git a/examples/safer_cluster/README.md b/examples/safer_cluster/README.md index 2e925ac85b..e400641ab9 100644 --- a/examples/safer_cluster/README.md +++ b/examples/safer_cluster/README.md @@ -22,6 +22,7 @@ This example illustrates how to instantiate the opinionated Safer Cluster module | location | | | master\_kubernetes\_version | Kubernetes version of the master | | network\_name | The name of the VPC being created | +| project\_id | The project ID the cluster is in | | region | The region in which the cluster resides | | service\_account | The service account to default running nodes as if not overridden in `node_pools`. | | subnet\_names | The names of the subnet being created | diff --git a/examples/safer_cluster/outputs.tf b/examples/safer_cluster/outputs.tf index 72e09c0467..d2f9706f02 100644 --- a/examples/safer_cluster/outputs.tf +++ b/examples/safer_cluster/outputs.tf @@ -69,3 +69,8 @@ output "zones" { description = "List of zones in which the cluster resides" value = module.gke.zones } + +output "project_id" { + description = "The project ID the cluster is in" + value = var.project_id +} diff --git a/test/fixtures/deploy_service/example.tf b/test/fixtures/deploy_service/example.tf index 421c551489..60904163a8 100644 --- a/test/fixtures/deploy_service/example.tf +++ b/test/fixtures/deploy_service/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/deploy_service" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/deploy_service/network.tf b/test/fixtures/deploy_service/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/deploy_service/network.tf +++ b/test/fixtures/deploy_service/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/disable_client_cert/example.tf b/test/fixtures/disable_client_cert/example.tf index 23ea6da936..5bacef9fb9 100644 --- a/test/fixtures/disable_client_cert/example.tf +++ b/test/fixtures/disable_client_cert/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/disable_client_cert" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name - network_project_id = var.project_id + network_project_id = var.project_ids[0] subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/disable_client_cert/network.tf b/test/fixtures/disable_client_cert/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/disable_client_cert/network.tf +++ b/test/fixtures/disable_client_cert/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/node_pool/example.tf b/test/fixtures/node_pool/example.tf index 28d51f2357..82dd01035c 100644 --- a/test/fixtures/node_pool/example.tf +++ b/test/fixtures/node_pool/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/node_pool" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,6 +25,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/node_pool/network.tf b/test/fixtures/node_pool/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/node_pool/network.tf +++ b/test/fixtures/node_pool/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/node_pool_update_variant/example.tf b/test/fixtures/node_pool_update_variant/example.tf index c3a21df3d5..b7f9c8c390 100644 --- a/test/fixtures/node_pool_update_variant/example.tf +++ b/test/fixtures/node_pool_update_variant/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/node_pool_update_variant" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,5 +25,5 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/node_pool_update_variant/network.tf b/test/fixtures/node_pool_update_variant/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/node_pool_update_variant/network.tf +++ b/test/fixtures/node_pool_update_variant/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/private_zonal_with_networking/example.tf b/test/fixtures/private_zonal_with_networking/example.tf index 23612f1610..87bf06ace4 100644 --- a/test/fixtures/private_zonal_with_networking/example.tf +++ b/test/fixtures/private_zonal_with_networking/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/private_zonal_with_networking" - project_id = var.project_id + project_id = var.project_ids[0] region = var.region zones = var.zones } diff --git a/test/fixtures/private_zonal_with_networking/outputs.tf b/test/fixtures/private_zonal_with_networking/outputs.tf index 08f9a8a2e8..4f9edf51a0 100644 --- a/test/fixtures/private_zonal_with_networking/outputs.tf +++ b/test/fixtures/private_zonal_with_networking/outputs.tf @@ -16,7 +16,7 @@ output "project_id" { - value = var.project_id + value = module.example.project_id } output "location" { diff --git a/test/fixtures/private_zonal_with_networking/variables.tf b/test/fixtures/private_zonal_with_networking/variables.tf index d610f1e807..5719be3c18 100644 --- a/test/fixtures/private_zonal_with_networking/variables.tf +++ b/test/fixtures/private_zonal_with_networking/variables.tf @@ -14,7 +14,8 @@ * limitations under the License. */ -variable "project_id" { +variable "project_ids" { + type = list(string) description = "The project ID to host the cluster in" } diff --git a/test/fixtures/safer_cluster/example.tf b/test/fixtures/safer_cluster/example.tf index a130769b42..8cd7983c27 100644 --- a/test/fixtures/safer_cluster/example.tf +++ b/test/fixtures/safer_cluster/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/safer_cluster" - project_id = var.project_id + project_id = var.project_ids[0] region = var.region - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/safer_cluster/outputs.tf b/test/fixtures/safer_cluster/outputs.tf index 86b0648afd..6613bcb29a 100644 --- a/test/fixtures/safer_cluster/outputs.tf +++ b/test/fixtures/safer_cluster/outputs.tf @@ -15,7 +15,7 @@ */ output "project_id" { - value = var.project_id + value = module.example.project_id } output "region" { diff --git a/test/fixtures/sandbox_enabled/example.tf b/test/fixtures/sandbox_enabled/example.tf index 05b7edfd9e..920c5e179f 100644 --- a/test/fixtures/sandbox_enabled/example.tf +++ b/test/fixtures/sandbox_enabled/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/simple_regional_beta" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] istio = false cloudrun = false node_metadata = "UNSPECIFIED" diff --git a/test/fixtures/sandbox_enabled/network.tf b/test/fixtures/sandbox_enabled/network.tf index 5d34d43748..f170a88fa4 100644 --- a/test/fixtures/sandbox_enabled/network.tf +++ b/test/fixtures/sandbox_enabled/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 71b4b250de..dfc0d02396 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -15,7 +15,7 @@ */ output "project_id" { - value = var.project_id + value = module.example.project_id } output "region" { diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index 9760d65a94..56e5b63441 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -14,8 +14,9 @@ * limitations under the License. */ -variable "project_id" { - description = "The GCP project to use for integration tests" +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" } variable "region" { @@ -29,8 +30,9 @@ variable "zones" { default = ["us-central1-a", "us-central1-b", "us-central1-c"] } -variable "compute_engine_service_account" { - description = "The email address of the service account to associate with the GKE cluster" +variable "compute_engine_service_accounts" { + type = list(string) + description = "The email addresses of the service account to associate with the GKE cluster" } variable "registry_project_id" { diff --git a/test/fixtures/shared_vpc/example.tf b/test/fixtures/shared_vpc/example.tf index 051d9155c9..a91d50fe85 100644 --- a/test/fixtures/shared_vpc/example.tf +++ b/test/fixtures/shared_vpc/example.tf @@ -17,14 +17,14 @@ module "example" { source = "../../../examples/shared_vpc" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name - network_project_id = var.project_id + network_project_id = var.project_ids[0] subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] } diff --git a/test/fixtures/shared_vpc/network.tf b/test/fixtures/shared_vpc/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/shared_vpc/network.tf +++ b/test/fixtures/shared_vpc/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index 7f8bb83637..2e08ae6628 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/simple_regional" - project_id = var.project_id + project_id = var.project_ids[0] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[0] skip_provisioners = true } diff --git a/test/fixtures/simple_regional/network.tf b/test/fixtures/simple_regional/network.tf index e1292eae3b..94bb29e63c 100644 --- a/test/fixtures/simple_regional/network.tf +++ b/test/fixtures/simple_regional/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[0] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_regional_private/example.tf b/test/fixtures/simple_regional_private/example.tf index c06f080ac4..e7c71a3c64 100644 --- a/test/fixtures/simple_regional_private/example.tf +++ b/test/fixtures/simple_regional_private/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/simple_regional_private" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/simple_regional_private/network.tf b/test/fixtures/simple_regional_private/network.tf index f34f629069..8d643281e1 100644 --- a/test/fixtures/simple_regional_private/network.tf +++ b/test/fixtures/simple_regional_private/network.tf @@ -21,13 +21,13 @@ resource "random_string" "suffix" { } resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/fixtures/simple_regional_with_networking/example.tf b/test/fixtures/simple_regional_with_networking/example.tf index c7ae5af76c..d8d2172341 100644 --- a/test/fixtures/simple_regional_with_networking/example.tf +++ b/test/fixtures/simple_regional_with_networking/example.tf @@ -17,6 +17,6 @@ module "example" { source = "../../../examples/simple_regional_with_networking" - project_id = var.project_id + project_id = var.project_ids[1] region = var.region } diff --git a/test/fixtures/simple_regional_with_networking/outputs.tf b/test/fixtures/simple_regional_with_networking/outputs.tf index 08f9a8a2e8..4f9edf51a0 100644 --- a/test/fixtures/simple_regional_with_networking/outputs.tf +++ b/test/fixtures/simple_regional_with_networking/outputs.tf @@ -16,7 +16,7 @@ output "project_id" { - value = var.project_id + value = module.example.project_id } output "location" { diff --git a/test/fixtures/simple_regional_with_networking/variables.tf b/test/fixtures/simple_regional_with_networking/variables.tf index e9310a56c5..9bbb43040c 100644 --- a/test/fixtures/simple_regional_with_networking/variables.tf +++ b/test/fixtures/simple_regional_with_networking/variables.tf @@ -14,8 +14,9 @@ * limitations under the License. */ -variable "project_id" { - description = "The project ID to host the cluster in" +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" } variable "region" { diff --git a/test/fixtures/simple_zonal/example.tf b/test/fixtures/simple_zonal/example.tf index 7d558e36f0..5498b28b00 100644 --- a/test/fixtures/simple_zonal/example.tf +++ b/test/fixtures/simple_zonal/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/simple_zonal_with_acm" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) diff --git a/test/fixtures/simple_zonal/network.tf b/test/fixtures/simple_zonal/network.tf index a4978c9ac3..e0bf46c2f2 100644 --- a/test/fixtures/simple_zonal/network.tf +++ b/test/fixtures/simple_zonal/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/simple_zonal_private/example.tf b/test/fixtures/simple_zonal_private/example.tf index 0f8a4fc478..3ccce8ceab 100644 --- a/test/fixtures/simple_zonal_private/example.tf +++ b/test/fixtures/simple_zonal_private/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/simple_zonal_private" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) @@ -25,6 +25,6 @@ module "example" { subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/simple_zonal_private/network.tf b/test/fixtures/simple_zonal_private/network.tf index 76d33f6bfc..fc7de87ab4 100644 --- a/test/fixtures/simple_zonal_private/network.tf +++ b/test/fixtures/simple_zonal_private/network.tf @@ -22,13 +22,13 @@ resource "random_string" "suffix" { resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/fixtures/stub_domains/example.tf b/test/fixtures/stub_domains/example.tf index 32cb31460e..df31547535 100644 --- a/test/fixtures/stub_domains/example.tf +++ b/test/fixtures/stub_domains/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/stub_domains" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/stub_domains/network.tf b/test/fixtures/stub_domains/network.tf index e1292eae3b..a24129ec4f 100644 --- a/test/fixtures/stub_domains/network.tf +++ b/test/fixtures/stub_domains/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/stub_domains_private/main.tf b/test/fixtures/stub_domains_private/main.tf index 101aac220a..a98f8d6ba6 100644 --- a/test/fixtures/stub_domains_private/main.tf +++ b/test/fixtures/stub_domains_private/main.tf @@ -24,7 +24,7 @@ resource "google_compute_network" "main" { name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false - project = var.project_id + project = var.project_ids[1] } resource "google_compute_subnetwork" "main" { @@ -32,7 +32,7 @@ resource "google_compute_subnetwork" "main" { name = "cft-gke-test-${random_string.suffix.result}" network = google_compute_network.main.self_link - project = var.project_id + project = var.project_ids[1] region = var.region secondary_ip_range { @@ -49,11 +49,11 @@ resource "google_compute_subnetwork" "main" { module "example" { source = "../../../examples/stub_domains_private" - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name network = google_compute_network.main.name - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region subnetwork = google_compute_subnetwork.main.name diff --git a/test/fixtures/stub_domains_upstream_nameservers/example.tf b/test/fixtures/stub_domains_upstream_nameservers/example.tf index a066994219..a6923c0f13 100644 --- a/test/fixtures/stub_domains_upstream_nameservers/example.tf +++ b/test/fixtures/stub_domains_upstream_nameservers/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/stub_domains_upstream_nameservers" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/stub_domains_upstream_nameservers/network.tf b/test/fixtures/stub_domains_upstream_nameservers/network.tf index ee202615ea..8ec5389ade 100644 --- a/test/fixtures/stub_domains_upstream_nameservers/network.tf +++ b/test/fixtures/stub_domains_upstream_nameservers/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/upstream_nameservers/example.tf b/test/fixtures/upstream_nameservers/example.tf index 72e6307fb1..81d60e8559 100644 --- a/test/fixtures/upstream_nameservers/example.tf +++ b/test/fixtures/upstream_nameservers/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/upstream_nameservers" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region network = google_compute_network.main.name subnetwork = google_compute_subnetwork.main.name ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + compute_engine_service_account = var.compute_engine_service_accounts[1] } diff --git a/test/fixtures/upstream_nameservers/network.tf b/test/fixtures/upstream_nameservers/network.tf index ee202615ea..8ec5389ade 100644 --- a/test/fixtures/upstream_nameservers/network.tf +++ b/test/fixtures/upstream_nameservers/network.tf @@ -21,7 +21,7 @@ resource "random_string" "suffix" { } provider "google" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { diff --git a/test/fixtures/workload_metadata_config/example.tf b/test/fixtures/workload_metadata_config/example.tf index 3568cfa404..ea9519579d 100644 --- a/test/fixtures/workload_metadata_config/example.tf +++ b/test/fixtures/workload_metadata_config/example.tf @@ -17,7 +17,7 @@ module "example" { source = "../../../examples/workload_metadata_config" - project_id = var.project_id + project_id = var.project_ids[1] cluster_name_suffix = "-${random_string.suffix.result}" region = var.region zones = slice(var.zones, 0, 1) diff --git a/test/fixtures/workload_metadata_config/network.tf b/test/fixtures/workload_metadata_config/network.tf index f163eb7730..f8a3322a65 100644 --- a/test/fixtures/workload_metadata_config/network.tf +++ b/test/fixtures/workload_metadata_config/network.tf @@ -21,17 +21,17 @@ resource "random_string" "suffix" { } provider "google-beta" { - project = var.project_id + project = var.project_ids[1] } resource "google_compute_network" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = "false" } resource "google_compute_subnetwork" "main" { - project = var.project_id + project = var.project_ids[1] name = "cft-gke-test-${random_string.suffix.result}" ip_cidr_range = "10.0.0.0/17" region = var.region diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 7ff4de74bc..b26c46568e 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -36,21 +36,35 @@ resource "random_id" "random_suffix" { } resource "google_service_account" "int_test" { - project = module.gke-project.project_id + project = module.gke-project-1.project_id account_id = "gke-int-test-${random_id.random_suffix.hex}" display_name = "gke-int-test" } -resource "google_service_account" "gke_sa" { - project = module.gke-project.project_id - account_id = "gke-sa-int-test-${random_id.random_suffix.hex}" - display_name = "gke-sa-int-test" +resource "google_service_account" "gke_sa_1" { + project = module.gke-project-1.project_id + account_id = "gke-sa-int-test-p1-${random_id.random_suffix.hex}" + display_name = "gke-sa-int-test-p1" } -resource "google_project_iam_member" "int_test" { +resource "google_service_account" "gke_sa_2" { + project = module.gke-project-2.project_id + account_id = "gke-sa-int-test-p2-${random_id.random_suffix.hex}" + display_name = "gke-sa-int-test-p2" +} + +resource "google_project_iam_member" "int_test_1" { + count = length(local.int_required_roles) + + project = module.gke-project-1.project_id + role = local.int_required_roles[count.index] + member = "serviceAccount:${google_service_account.int_test.email}" +} + +resource "google_project_iam_member" "int_test_2" { count = length(local.int_required_roles) - project = module.gke-project.project_id + project = module.gke-project-2.project_id role = local.int_required_roles[count.index] member = "serviceAccount:${google_service_account.int_test.email}" } diff --git a/test/setup/main.tf b/test/setup/main.tf index 70e10c46a3..961d1b47ac 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -14,7 +14,35 @@ * limitations under the License. */ -module "gke-project" { +module "gke-project-1" { + source = "terraform-google-modules/project-factory/google" + version = "~> 3.0" + + name = "ci-gke" + random_project_id = true + org_id = var.org_id + folder_id = var.folder_id + billing_account = var.billing_account + + auto_create_network = true + + activate_apis = [ + "bigquery-json.googleapis.com", + "cloudkms.googleapis.com", + "cloudresourcemanager.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "containerregistry.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "oslogin.googleapis.com", + "pubsub.googleapis.com", + "serviceusage.googleapis.com", + "storage-api.googleapis.com", + ] +} + +module "gke-project-2" { source = "terraform-google-modules/project-factory/google" version = "~> 3.0" diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index 3e508ed1c7..b0f9877276 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -14,8 +14,8 @@ * limitations under the License. */ -output "project_id" { - value = module.gke-project.project_id +output "project_ids" { + value = [module.gke-project-1.project_id, module.gke-project-2.project_id] } output "sa_key" { @@ -23,6 +23,6 @@ output "sa_key" { sensitive = true } -output "compute_engine_service_account" { - value = google_service_account.gke_sa.email +output "compute_engine_service_accounts" { + value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email] } From 0be30c8fd6acd46684d830b236e1806c17ba3897 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Tue, 26 Nov 2019 00:05:16 -0600 Subject: [PATCH 062/110] fix outputs --- test/setup/make_source.sh | 12 ++++++------ test/setup/outputs.tf | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh index ad3f57165a..4af7cd63cd 100755 --- a/test/setup/make_source.sh +++ b/test/setup/make_source.sh @@ -16,15 +16,15 @@ echo "#!/usr/bin/env bash" > ../source.sh -project_id=$(terraform output project_id) -echo "export TF_VAR_project_id='$project_id'" >> ../source.sh +project_ids=$(terraform output project_ids) +echo "export TF_VAR_project_ids='$project_ids'" >> ../source.sh -# We use the same project for registry project in the tests. -echo "export TF_VAR_registry_project_id='$project_id'" >> ../source.sh +registry_project_id=$(terraform output registry_project_id) +echo "export TF_VAR_registry_project_id='$registry_project_id'" >> ../source.sh sa_json=$(terraform output sa_key) # shellcheck disable=SC2086 echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh -compute_engine_service_account=$(terraform output compute_engine_service_account) -echo "export TF_VAR_compute_engine_service_account='$compute_engine_service_account'" >> ../source.sh +compute_engine_service_accounts=$(terraform output compute_engine_service_accounts) +echo "export TF_VAR_compute_engine_service_accounts='$compute_engine_service_accounts'" >> ../source.sh diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index b0f9877276..ddebd75861 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -26,3 +26,7 @@ output "sa_key" { output "compute_engine_service_accounts" { value = [google_service_account.gke_sa_1.email, google_service_account.gke_sa_2.email] } + +output "registry_project_id" { + value = module.gke-project-1.project_id +} From 7def7571f036fa84a01467e81656023d74a97da3 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 26 Nov 2019 12:52:48 -0500 Subject: [PATCH 063/110] Add multiple issues to CHANGELOG #315, #195, #271, #339, #290, #308, #321, #261 --- CHANGELOG.md | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a849f89183..20b4477765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,12 +14,30 @@ Extending the adopted spec, each change should have a link to its corresponding * Support for setting node_locations on node pools. [#303] * Fix for specifying `node_count` on node pools when autoscaling is disabled. [#311] * Added submodule for installing Anthos Config Management. [#268] -* Support for `local_ssd_count` in node pool configuration. [#244] +* Support for `local_ssd_count` in node pool configuration. [#339] * Wait for cluster to be ready before returning endpoint. [#340] +* `safer-cluster` submodule. [#315] +* `simple_regional_with_networking` example. [#195] +* `release_channel` variable for beta submodules. [#271] +* The `node_locations` attribute to the `node_pools` object for beta submodules. [#290] +* `private_zonal_with_nteworking` example. [#308] +* `regional_private_node_pool_oauth_scopes` example. [#321] + +### Changed + +* The `node_pool_labels`, `node_pool_tags`, and `node_pool_taints` variables have defaults and can be overridden within the + `node_pools` object. [#3] +* `upstream_nameservers` variable is typed as a list of strings. [#350] ### Removed * **Breaking**: Removed support for enabling the Kubernetes dashboard, as this is deprecated on GKE. [#337] +* **Beaking**: Removed support for versions of the Google provider and the Google Beta provider older than 2.18. [#261] + +### Fixed + +* `identity_namespace` output depends on the `google_container_cluster.primary` resource. [#301] +* Idempotency of the beta submodules. [#326] ## [v5.1.1] - 2019-10-25 @@ -217,7 +235,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o * Initial release of module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.2.0...HEAD +[v5.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...v5.2.0 [v5.1.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...v5.1.1 [v5.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...v5.1.0 [v5.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.1.0...v5.0.0 @@ -234,17 +253,27 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 -[#337]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/337 +[#350]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/350 [#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 -[#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268 +[#339]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/339 +[#337]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/337 +[#326]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/326 +[#321]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/321 +[#315]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/315 [#311]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/311 +[#308]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/308 [#303]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/303 +[#301]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/301 [#300]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/300 +[#290]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/290 [#286]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/286 [#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285 [#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284 [#282]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/282 [#273]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/273 +[#271]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/271 +[#268]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/268 +[#261]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/261 [#258]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/258 [#256]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/256 [#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248 @@ -253,7 +282,6 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238 [#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241 [#250]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/250 -[#244]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/244 [#236]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/236 [#217]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/217 [#234]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/234 @@ -265,6 +293,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#203]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/203 [#198]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/198 [#197]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/197 +[#195]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/195 [#193]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/193 [#188]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/188 [#187]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/187 @@ -312,6 +341,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#15]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/15 [#10]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/10 [#9]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/9 +[#3]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/3 [upgrading-to-v2.0]: docs/upgrading_to_v2.0.md [upgrading-to-v3.0]: docs/upgrading_to_v3.0.md From d856f2b7124759e8adb6e8d1a460c53c05d4b077 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 26 Nov 2019 14:40:38 -0500 Subject: [PATCH 064/110] Remove trailing whitespace from CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20b4477765..d75c2ce5b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ Extending the adopted spec, each change should have a link to its corresponding ### Changed -* The `node_pool_labels`, `node_pool_tags`, and `node_pool_taints` variables have defaults and can be overridden within the +* The `node_pool_labels`, `node_pool_tags`, and `node_pool_taints` variables have defaults and can be overridden within the `node_pools` object. [#3] * `upstream_nameservers` variable is typed as a list of strings. [#350] From c61e893b26a123044f6b8da60b75b1fa4954cb8d Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Tue, 26 Nov 2019 16:38:17 -0500 Subject: [PATCH 065/110] Add #93 to CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d75c2ce5b7..aca3cd5f8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,9 @@ Extending the adopted spec, each change should have a link to its corresponding * `simple_regional_with_networking` example. [#195] * `release_channel` variable for beta submodules. [#271] * The `node_locations` attribute to the `node_pools` object for beta submodules. [#290] -* `private_zonal_with_nteworking` example. [#308] +* `private_zonal_with_networking` example. [#308] * `regional_private_node_pool_oauth_scopes` example. [#321] +* The `cluster_autoscaling` variable for beta submodules. [#93] ### Changed @@ -314,6 +315,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#108]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/108 [#106]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/106 [#94]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/94 +[#93]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/93 [#89]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/89 [#80]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/80 [#77]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/77 From 8309664133bad009bd23c75add5ebe210e1141b1 Mon Sep 17 00:00:00 2001 From: Pavel Zhukov Date: Wed, 27 Nov 2019 13:49:51 +0200 Subject: [PATCH 066/110] Remove make_source.sh. Outputs are automatically piped to kitchen_do --- test/setup/make_source.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100755 test/setup/make_source.sh diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh deleted file mode 100755 index 4af7cd63cd..0000000000 --- a/test/setup/make_source.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -echo "#!/usr/bin/env bash" > ../source.sh - -project_ids=$(terraform output project_ids) -echo "export TF_VAR_project_ids='$project_ids'" >> ../source.sh - -registry_project_id=$(terraform output registry_project_id) -echo "export TF_VAR_registry_project_id='$registry_project_id'" >> ../source.sh - -sa_json=$(terraform output sa_key) -# shellcheck disable=SC2086 -echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh - -compute_engine_service_accounts=$(terraform output compute_engine_service_accounts) -echo "export TF_VAR_compute_engine_service_accounts='$compute_engine_service_accounts'" >> ../source.sh From 94496f9970790566f49143ff3a5bf1fd18827efa Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 27 Nov 2019 17:43:05 -0500 Subject: [PATCH 067/110] Simplify master_authorized_networks_config to master_authorized_networks --- README.md | 2 +- autogen/cluster.tf.tmpl | 2 +- autogen/main.tf.tmpl | 4 ++++ autogen/outputs.tf.tmpl | 2 +- autogen/variables.tf.tmpl | 6 +++--- cluster.tf | 2 +- examples/node_pool_update_variant/main.tf | 10 +++------- examples/node_pool_update_variant_beta/main.tf | 10 +++------- examples/private_zonal_with_networking/main.tf | 10 +++------- .../regional_private_node_pool_oauth_scopes/main.tf | 10 +++------- examples/safer_cluster/main.tf | 12 +++++------- examples/simple_regional_private/main.tf | 10 +++------- examples/simple_regional_private_beta/main.tf | 10 +++------- examples/simple_zonal_private/main.tf | 10 +++------- examples/stub_domains_private/main.tf | 10 +++------- examples/workload_metadata_config/main.tf | 10 +++------- main.tf | 4 ++++ .../beta-private-cluster-update-variant/README.md | 2 +- .../beta-private-cluster-update-variant/cluster.tf | 2 +- modules/beta-private-cluster-update-variant/main.tf | 4 ++++ .../beta-private-cluster-update-variant/outputs.tf | 2 +- .../beta-private-cluster-update-variant/variables.tf | 6 +++--- modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/cluster.tf | 2 +- modules/beta-private-cluster/main.tf | 4 ++++ modules/beta-private-cluster/outputs.tf | 2 +- modules/beta-private-cluster/variables.tf | 6 +++--- modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/cluster.tf | 2 +- modules/beta-public-cluster/main.tf | 4 ++++ modules/beta-public-cluster/outputs.tf | 2 +- modules/beta-public-cluster/variables.tf | 6 +++--- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster-update-variant/cluster.tf | 2 +- modules/private-cluster-update-variant/main.tf | 4 ++++ modules/private-cluster-update-variant/outputs.tf | 2 +- modules/private-cluster-update-variant/variables.tf | 6 +++--- modules/private-cluster/README.md | 2 +- modules/private-cluster/cluster.tf | 2 +- modules/private-cluster/main.tf | 4 ++++ modules/private-cluster/outputs.tf | 2 +- modules/private-cluster/variables.tf | 6 +++--- modules/safer-cluster/README.md | 2 +- modules/safer-cluster/main.tf | 2 +- modules/safer-cluster/variables.tf | 6 +++--- outputs.tf | 2 +- variables.tf | 6 +++--- 47 files changed, 106 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index 4c15b0f311..1cebf678c5 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | diff --git a/autogen/cluster.tf.tmpl b/autogen/cluster.tf.tmpl index 1ca0759a47..dc19ed2d90 100644 --- a/autogen/cluster.tf.tmpl +++ b/autogen/cluster.tf.tmpl @@ -101,7 +101,7 @@ resource "google_container_cluster" "primary" { } {% endif %} dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 629a47729b..3ac28cc16a 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -127,6 +127,10 @@ locals { # /BETA features {% endif %} + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/autogen/outputs.tf.tmpl b/autogen/outputs.tf.tmpl index 3db3358196..4a2df6f198 100644 --- a/autogen/outputs.tf.tmpl +++ b/autogen/outputs.tf.tmpl @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 3b20d3140b..3295199e48 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/cluster.tf b/cluster.tf index 70d932ccfe..573b4cc5d8 100644 --- a/cluster.tf +++ b/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/examples/node_pool_update_variant/main.tf b/examples/node_pool_update_variant/main.tf index 9b29a5f0fe..d6f7f270b7 100644 --- a/examples/node_pool_update_variant/main.tf +++ b/examples/node_pool_update_variant/main.tf @@ -46,14 +46,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/node_pool_update_variant_beta/main.tf b/examples/node_pool_update_variant_beta/main.tf index 37b595f793..a127e9aae2 100644 --- a/examples/node_pool_update_variant_beta/main.tf +++ b/examples/node_pool_update_variant_beta/main.tf @@ -47,14 +47,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf index 82db39a591..b7bb59c9d2 100644 --- a/examples/private_zonal_with_networking/main.tf +++ b/examples/private_zonal_with_networking/main.tf @@ -71,14 +71,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/regional_private_node_pool_oauth_scopes/main.tf b/examples/regional_private_node_pool_oauth_scopes/main.tf index e39e3299f6..b843cb7234 100644 --- a/examples/regional_private_node_pool_oauth_scopes/main.tf +++ b/examples/regional_private_node_pool_oauth_scopes/main.tf @@ -33,14 +33,10 @@ module "gke" { remove_default_node_pool = true disable_legacy_metadata_endpoints = true - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = module.gke-network.subnets_ips[0] - display_name = "VPC" - }, - ] + cidr_block = module.gke-network.subnets_ips[0] + display_name = "VPC" }, ] diff --git a/examples/safer_cluster/main.tf b/examples/safer_cluster/main.tf index 0aeca18009..a69cc3ddbb 100644 --- a/examples/safer_cluster/main.tf +++ b/examples/safer_cluster/main.tf @@ -49,16 +49,14 @@ module "gke" { ip_range_services = local.svc_range_name compute_engine_service_account = var.compute_engine_service_account master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = "10.60.0.0/17" - display_name = "VPC" - }, - ] + cidr_block = "10.60.0.0/17" + display_name = "VPC" }, ] + istio = true cloudrun = true } diff --git a/examples/simple_regional_private/main.tf b/examples/simple_regional_private/main.tf index f17a3728a6..10a55795df 100644 --- a/examples/simple_regional_private/main.tf +++ b/examples/simple_regional_private/main.tf @@ -45,14 +45,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/simple_regional_private_beta/main.tf b/examples/simple_regional_private_beta/main.tf index 4e1d405940..eec167a3e4 100644 --- a/examples/simple_regional_private_beta/main.tf +++ b/examples/simple_regional_private_beta/main.tf @@ -44,14 +44,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/simple_zonal_private/main.tf b/examples/simple_zonal_private/main.tf index ae1a90a6cc..0b2a3c4180 100644 --- a/examples/simple_zonal_private/main.tf +++ b/examples/simple_zonal_private/main.tf @@ -46,14 +46,10 @@ module "gke" { enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf index 3f268e75a4..b263922b2a 100644 --- a/examples/stub_domains_private/main.tf +++ b/examples/stub_domains_private/main.tf @@ -40,14 +40,10 @@ module "gke" { enable_private_endpoint = false enable_private_nodes = true - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf index 3d2254c2da..fbcfc949ae 100644 --- a/examples/workload_metadata_config/main.tf +++ b/examples/workload_metadata_config/main.tf @@ -48,14 +48,10 @@ module "gke" { master_ipv4_cidr_block = "172.16.0.0/28" node_metadata = "SECURE" - master_authorized_networks_config = [ + master_authorized_networks = [ { - cidr_blocks = [ - { - cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range - display_name = "VPC" - }, - ] + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" }, ] } diff --git a/main.tf b/main.tf index 754fcefe04..7dd9d55171 100644 --- a/main.tf +++ b/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks: var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 1f4e4ea298..6df69df5bd 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -171,7 +171,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 52961dfd86..10fcf764a3 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -93,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 02d84e73ab..9afc2502b8 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -113,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index d3f7504667..62c47d002d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 528819506e..b03a4ea921 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -171,7 +171,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 604fc7b2d4..f2789bfd03 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -93,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 02d84e73ab..9afc2502b8 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -113,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d3f7504667..62c47d002d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index e7d7820595..b6ab1f39d3 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -163,7 +163,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index cb15fe91c6..5867718699 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -93,7 +93,7 @@ resource "google_container_cluster" "primary" { } } dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 6200b42633..5cff8bdd4e 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -113,6 +113,10 @@ locals { # /BETA features + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks : var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 32e37ef5a3..8ea1ded043 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index dc25c7a434..1a1b9c54b4 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 1786d3b46c..8d26153072 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -157,7 +157,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 27653ab689..7027d766ce 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 7826dfff18..9861987f40 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks: var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 1099fe5f95..9c75edb5ec 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 0f5c3df935..60fbe8de76 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -157,7 +157,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 47b50f6db2..acb6f29a68 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { monitoring_service = var.monitoring_service dynamic "master_authorized_networks_config" { - for_each = var.master_authorized_networks_config + for_each = local.master_authorized_networks_config content { dynamic "cidr_blocks" { for_each = master_authorized_networks_config.value.cidr_blocks diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 7826dfff18..9861987f40 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -82,6 +82,10 @@ locals { cluster_output_horizontal_pod_autoscaling_enabled = google_container_cluster.primary.addons_config.0.horizontal_pod_autoscaling.0.disabled + master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ + cidr_blocks: var.master_authorized_networks + }] + cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) cluster_output_node_pools_versions = concat(google_container_node_pool.pools.*.version, [""]) diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 1099fe5f95..9c75edb5ec 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 2d99d26f36..06e81e12c9 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -221,7 +221,7 @@ For simplicity, we suggest using `roles/container.admin` and | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. The module enforces certain minimum versions to ensure that specific features are available. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster. | object | `` | no | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster | string | n/a | yes | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 76c40b3ebe..4b2c00109d 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -36,7 +36,7 @@ module "gke" { // https://cloud.google.com/kubernetes-engine/versioning-and-upgrades node_version = "" - master_authorized_networks_config = var.master_authorized_networks_config + master_authorized_networks = var.master_authorized_networks subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index fe47be7be6..40fa320dc6 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -77,9 +77,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "Additional CIDR of private networks that can access the master. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. By default, the private master endpoint is accessible by the nodes in the cluster's VPC and by Google's internal production jobs managing the cluster." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } diff --git a/outputs.tf b/outputs.tf index 0a4e6c9ada..6d5e1e3909 100644 --- a/outputs.tf +++ b/outputs.tf @@ -75,7 +75,7 @@ output "monitoring_service" { output "master_authorized_networks_config" { description = "Networks from which access to master is permitted" - value = var.master_authorized_networks_config + value = google_container_cluster.primary.master_authorized_networks_config } output "master_version" { diff --git a/variables.tf b/variables.tf index 42dbc421d1..904cd6ddab 100644 --- a/variables.tf +++ b/variables.tf @@ -78,9 +78,9 @@ variable "node_version" { default = "" } -variable "master_authorized_networks_config" { - type = list(object({ cidr_blocks = list(object({ cidr_block = string, display_name = string })) })) - description = "The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." default = [] } From e92a99ae2e0e2a1360f22367cf554f3c45f3257a Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 27 Nov 2019 17:47:42 -0500 Subject: [PATCH 068/110] Start v6.0 upgrade guide --- docs/upgrading_to_v6.0.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 docs/upgrading_to_v6.0.md diff --git a/docs/upgrading_to_v6.0.md b/docs/upgrading_to_v6.0.md new file mode 100644 index 0000000000..c0dad7cffb --- /dev/null +++ b/docs/upgrading_to_v6.0.md @@ -0,0 +1,32 @@ +# Upgrading to v6.0 + +The v6.0 release of *kubernetes-engine* is a backwards incompatible +release. + +## Migration Instructions + +### Master Authorized Networks +Previously, setting up master authorized networks required setting a nested config within `master_authorized_networks_config`. +Now, to set up master authorized networks you can simply pass a list of authorized networks. + +```diff + module "kubernetes_engine_private_cluster" { + source = "terraform-google-modules/kubernetes-engine/google" +- version = "~> 5.0" ++ version = "~> 6.0" + +- master_authorized_networks_config = [ ++ master_authorized_networks = [ + { +- cidr_blocks = [ +- { +- cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range +- display_name = "VPC" +- }, +- ] ++ cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range ++ display_name = "VPC" + }, + ] + } +``` From dd81751140fc52e65a76e09c2a66d518fd618769 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 27 Nov 2019 17:49:51 -0500 Subject: [PATCH 069/110] add k8s note to upgrade doc --- docs/upgrading_to_v6.0.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/upgrading_to_v6.0.md b/docs/upgrading_to_v6.0.md index c0dad7cffb..b7e9b43374 100644 --- a/docs/upgrading_to_v6.0.md +++ b/docs/upgrading_to_v6.0.md @@ -3,6 +3,11 @@ The v6.0 release of *kubernetes-engine* is a backwards incompatible release. +## Dropped support +Due to changes in GKE, the module has dropped support for setting the `kubernetes_dashboard` variable. + +Additionally, support for Google provider versions older than v2.18 has been removed. + ## Migration Instructions ### Master Authorized Networks From 3d23898bd74fb10c46f1dc33d56e43adba49eaf8 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Wed, 27 Nov 2019 17:56:24 -0500 Subject: [PATCH 070/110] Fix terraform fmt --- main.tf | 2 +- modules/private-cluster-update-variant/main.tf | 2 +- modules/private-cluster/main.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/main.tf b/main.tf index 7dd9d55171..409cbfe55b 100644 --- a/main.tf +++ b/main.tf @@ -83,7 +83,7 @@ locals { master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ - cidr_blocks: var.master_authorized_networks + cidr_blocks : var.master_authorized_networks }] cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 9861987f40..7dea99f25c 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -83,7 +83,7 @@ locals { master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ - cidr_blocks: var.master_authorized_networks + cidr_blocks : var.master_authorized_networks }] cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 9861987f40..7dea99f25c 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -83,7 +83,7 @@ locals { master_authorized_networks_config = length(var.master_authorized_networks) == 0 ? [] : [{ - cidr_blocks: var.master_authorized_networks + cidr_blocks : var.master_authorized_networks }] cluster_output_node_pools_names = concat(google_container_node_pool.pools.*.name, [""]) From c7787f2d1b94636c98c8a3eccd464e98c2dff3b6 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 28 Nov 2019 13:31:43 -0600 Subject: [PATCH 071/110] enable network_policy by default, fix examples --- README.md | 2 +- autogen/variables.tf.tmpl | 2 +- examples/stub_domains/main.tf | 1 - examples/stub_domains_private/main.tf | 1 - examples/stub_domains_upstream_nameservers/main.tf | 1 - examples/upstream_nameservers/main.tf | 1 - modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster-update-variant/variables.tf | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-private-cluster/variables.tf | 2 +- modules/beta-public-cluster/README.md | 2 +- modules/beta-public-cluster/variables.tf | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster-update-variant/variables.tf | 2 +- modules/private-cluster/README.md | 2 +- modules/private-cluster/variables.tf | 2 +- variables.tf | 2 +- 17 files changed, 13 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 1cebf678c5..1112b0b348 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | diff --git a/autogen/variables.tf.tmpl b/autogen/variables.tf.tmpl index 3295199e48..9a94ee8c66 100644 --- a/autogen/variables.tf.tmpl +++ b/autogen/variables.tf.tmpl @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/examples/stub_domains/main.tf b/examples/stub_domains/main.tf index b81dc0cf8c..90ffd97687 100644 --- a/examples/stub_domains/main.tf +++ b/examples/stub_domains/main.tf @@ -32,7 +32,6 @@ module "gke" { subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services - network_policy = true service_account = var.compute_engine_service_account create_service_account = false diff --git a/examples/stub_domains_private/main.tf b/examples/stub_domains_private/main.tf index b263922b2a..31b3d7aec5 100644 --- a/examples/stub_domains_private/main.tf +++ b/examples/stub_domains_private/main.tf @@ -49,7 +49,6 @@ module "gke" { master_ipv4_cidr_block = "172.16.0.0/28" - network_policy = true create_service_account = false service_account = var.compute_engine_service_account diff --git a/examples/stub_domains_upstream_nameservers/main.tf b/examples/stub_domains_upstream_nameservers/main.tf index 4b7448b7e8..009de87950 100644 --- a/examples/stub_domains_upstream_nameservers/main.tf +++ b/examples/stub_domains_upstream_nameservers/main.tf @@ -32,7 +32,6 @@ module "gke" { subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services - network_policy = true create_service_account = false service_account = var.compute_engine_service_account diff --git a/examples/upstream_nameservers/main.tf b/examples/upstream_nameservers/main.tf index 784e8a0cd3..26895f32eb 100644 --- a/examples/upstream_nameservers/main.tf +++ b/examples/upstream_nameservers/main.tf @@ -32,7 +32,6 @@ module "gke" { subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services - network_policy = true create_service_account = false service_account = var.compute_engine_service_account diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 6df69df5bd..3577bac930 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -176,7 +176,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 62c47d002d..da7c358427 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index b03a4ea921..8a799ccd3f 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -176,7 +176,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 62c47d002d..da7c358427 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index b6ab1f39d3..f2df9a497f 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -167,7 +167,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 1a1b9c54b4..3b20891262 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 8d26153072..e143917f25 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -162,7 +162,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 9c75edb5ec..ff98b283b0 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 60fbe8de76..eed365a5c4 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -162,7 +162,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | +| network\_policy | Enable network policy addon | bool | `"true"` | no | | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 9c75edb5ec..ff98b283b0 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { diff --git a/variables.tf b/variables.tf index 904cd6ddab..9c420fb423 100644 --- a/variables.tf +++ b/variables.tf @@ -99,7 +99,7 @@ variable "http_load_balancing" { variable "network_policy" { type = bool description = "Enable network policy addon" - default = false + default = true } variable "network_policy_provider" { From 2abd7b442a181ec45d9901fc9d75063d872ad999 Mon Sep 17 00:00:00 2001 From: bharathkkb Date: Thu, 28 Nov 2019 13:58:41 -0600 Subject: [PATCH 072/110] check for network policy enabled instead of disabled in kitchen tests --- .../private_zonal_with_networking/controls/gcloud.rb | 4 +--- test/integration/sandbox_enabled/controls/gcloud.rb | 4 +--- test/integration/simple_regional/controls/gcloud.rb | 4 +--- test/integration/simple_regional_private/controls/gcloud.rb | 4 +--- .../simple_regional_with_networking/controls/gcloud.rb | 4 +--- test/integration/simple_zonal/controls/gcloud.rb | 4 +--- test/integration/simple_zonal_private/controls/gcloud.rb | 4 +--- 7 files changed, 7 insertions(+), 21 deletions(-) diff --git a/test/integration/private_zonal_with_networking/controls/gcloud.rb b/test/integration/private_zonal_with_networking/controls/gcloud.rb index adaf6fd646..6e24b1142c 100644 --- a/test/integration/private_zonal_with_networking/controls/gcloud.rb +++ b/test/integration/private_zonal_with_networking/controls/gcloud.rb @@ -58,9 +58,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/sandbox_enabled/controls/gcloud.rb b/test/integration/sandbox_enabled/controls/gcloud.rb index eb0ffdaf46..a5b785e725 100644 --- a/test/integration/sandbox_enabled/controls/gcloud.rb +++ b/test/integration/sandbox_enabled/controls/gcloud.rb @@ -50,9 +50,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/simple_regional/controls/gcloud.rb b/test/integration/simple_regional/controls/gcloud.rb index e6bbcfc047..0f47490d40 100644 --- a/test/integration/simple_regional/controls/gcloud.rb +++ b/test/integration/simple_regional/controls/gcloud.rb @@ -50,9 +50,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/simple_regional_private/controls/gcloud.rb b/test/integration/simple_regional_private/controls/gcloud.rb index b15dafcd02..b86834a2a2 100644 --- a/test/integration/simple_regional_private/controls/gcloud.rb +++ b/test/integration/simple_regional_private/controls/gcloud.rb @@ -54,9 +54,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/simple_regional_with_networking/controls/gcloud.rb b/test/integration/simple_regional_with_networking/controls/gcloud.rb index e6bbcfc047..0f47490d40 100644 --- a/test/integration/simple_regional_with_networking/controls/gcloud.rb +++ b/test/integration/simple_regional_with_networking/controls/gcloud.rb @@ -50,9 +50,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/simple_zonal/controls/gcloud.rb b/test/integration/simple_zonal/controls/gcloud.rb index c2e72936b0..058ed9ba53 100644 --- a/test/integration/simple_zonal/controls/gcloud.rb +++ b/test/integration/simple_zonal/controls/gcloud.rb @@ -55,9 +55,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end diff --git a/test/integration/simple_zonal_private/controls/gcloud.rb b/test/integration/simple_zonal_private/controls/gcloud.rb index 9968affcb6..653c11bb0c 100644 --- a/test/integration/simple_zonal_private/controls/gcloud.rb +++ b/test/integration/simple_zonal_private/controls/gcloud.rb @@ -58,9 +58,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, }) end end From d780442fc4b0ab8e3bcf18ec580a192a241fc244 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 28 Nov 2019 12:20:07 -0500 Subject: [PATCH 073/110] Remove upgrade notes from CHANGELOG This content of this section is incomplete (missing 5.0.0) and redundant with the CHANGELOG. --- README.md | 19 ------------- autogen/README.md | 27 ------------------- .../README.md | 19 ------------- modules/beta-private-cluster/README.md | 19 ------------- modules/beta-public-cluster/README.md | 19 ------------- .../private-cluster-update-variant/README.md | 19 ------------- modules/private-cluster/README.md | 19 ------------- 7 files changed, 141 deletions(-) diff --git a/README.md b/README.md index 1112b0b348..9e0bdc76d8 100644 --- a/README.md +++ b/README.md @@ -108,22 +108,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -251,9 +235,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: docs/upgrading_to_v3.0.md [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/autogen/README.md b/autogen/README.md index c8e956a76e..71f0f6d986 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -125,22 +125,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - @@ -199,17 +183,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -{% if private_cluster %} -[upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md -{% else %} -[upgrading-to-v2.0]: docs/upgrading_to_v2.0.md -{% endif %} -{% if private_cluster or beta_cluster %} -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md -{% else %} -[upgrading-to-v3.0]: docs/upgrading_to_v3.0.md -{% endif %} {% if beta_cluster %} [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta {% else %} diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 3577bac930..5ef2738358 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -116,22 +116,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -287,9 +271,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 8a799ccd3f..6794e9bf64 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -116,22 +116,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -287,9 +271,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index f2df9a497f..7fbc72a36f 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -111,22 +111,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -278,9 +262,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md [terraform-provider-google-beta]: https://github.com/terraform-providers/terraform-provider-google-beta [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index e143917f25..7732c5b000 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -113,22 +113,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -260,9 +244,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index eed365a5c4..ddfaa0199d 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -113,22 +113,6 @@ Then perform the following commands on the root folder: - `terraform apply` to apply the infrastructure build - `terraform destroy` to destroy the built infrastructure -## Upgrade to v3.0.0 - -v3.0.0 is a breaking release. Refer to the -[Upgrading to v3.0 guide][upgrading-to-v3.0] for details. - -## Upgrade to v2.0.0 - -v2.0.0 is a breaking release. Refer to the -[Upgrading to v2.0 guide][upgrading-to-v2.0] for details. - -## Upgrade to v1.0.0 - -Version 1.0.0 of this module introduces a breaking change: adding the `disable-legacy-endpoints` metadata field to all node pools. This metadata is required by GKE and [determines whether the `/0.1/` and `/v1beta1/` paths are available in the nodes' metadata server](https://cloud.google.com/kubernetes-engine/docs/how-to/protecting-cluster-metadata#disable-legacy-apis). If your applications do not require access to the node's metadata server, you can leave the default value of `true` provided by the module. If your applications require access to the metadata server, be sure to read the linked documentation to see if you need to set the value for this field to `false` to allow your applications access to the above metadata server paths. - -In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. - ## Inputs @@ -260,9 +244,6 @@ The project has the following folders and files: - /README.MD: This file. - /modules: Private and beta sub modules. - -[upgrading-to-v2.0]: ../../docs/upgrading_to_v2.0.md -[upgrading-to-v3.0]: ../../docs/upgrading_to_v3.0.md [terraform-provider-google]: https://github.com/terraform-providers/terraform-provider-google [3.0.0]: https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/3.0.0 [terraform-0.12-upgrade]: https://www.terraform.io/upgrade-guides/0-12.html From 97529bc0e96e1dd40941bbb6d06cc41dcf9da08a Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Thu, 28 Nov 2019 12:23:30 -0500 Subject: [PATCH 074/110] Add #354, 6.0.0 to CHANGELOG --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aca3cd5f8a..b2f2d2ec67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] +## [v6.0.0] - 2019-11-28 + +v6.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v6.0.md). + ### Added * Support for Shielded Nodes beta feature via `enabled_shielded_nodes` variable. [#300] @@ -23,6 +27,7 @@ Extending the adopted spec, each change should have a link to its corresponding * `private_zonal_with_networking` example. [#308] * `regional_private_node_pool_oauth_scopes` example. [#321] * The `cluster_autoscaling` variable for beta submodules. [#93] +* The `master_authorized_networks` variable. [#354] ### Changed @@ -34,6 +39,7 @@ Extending the adopted spec, each change should have a link to its corresponding * **Breaking**: Removed support for enabling the Kubernetes dashboard, as this is deprecated on GKE. [#337] * **Beaking**: Removed support for versions of the Google provider and the Google Beta provider older than 2.18. [#261] +* **Breaking**: Removed the `master_authorized_networks_config` variable. [#354] ### Fixed @@ -236,7 +242,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o * Initial release of module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.2.0...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.0...HEAD +[v6.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.2.0...v6.0.0 [v5.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...v5.2.0 [v5.1.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...v5.1.1 [v5.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...v5.1.0 @@ -254,6 +261,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#354]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/354 [#350]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/350 [#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 [#339]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/339 From d426a1eaac049e39010b692817c6c3d00e837a10 Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 29 Nov 2019 10:47:30 -0500 Subject: [PATCH 075/110] Add #138 to CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2f2d2ec67..9f60301859 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ v6.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./d * The `node_pool_labels`, `node_pool_tags`, and `node_pool_taints` variables have defaults and can be overridden within the `node_pools` object. [#3] * `upstream_nameservers` variable is typed as a list of strings. [#350] +* The `network_policy` variable defaults to `true`. [#138] ### Removed @@ -315,6 +316,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#151]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/151 [#149]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/149 [#148]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/148 +[#136]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/138 [#136]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/136 [#132]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/132 [#124]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/124 From 13a08a41dac1072e8b4729021e0982cf7f8cf6bf Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 29 Nov 2019 12:01:57 -0500 Subject: [PATCH 076/110] Update beta_cluster project configuration --- test/fixtures/beta_cluster/main.tf | 19 ++++--------------- test/fixtures/beta_cluster/network.tf | 3 ++- test/fixtures/beta_cluster/outputs.tf | 2 +- test/setup/iam.tf | 4 ++-- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf index 4431b715f3..58941b7f69 100644 --- a/test/fixtures/beta_cluster/main.tf +++ b/test/fixtures/beta_cluster/main.tf @@ -14,26 +14,15 @@ * limitations under the License. */ - -provider "google" { - version = "~> 2.18.0" - project = var.project_id - region = var.region -} - -provider "google-beta" { - version = "~> 2.18.0" - project = var.project_id - region = var.region -} - locals { - name = "beta-cluster-${random_string.suffix.result}" + name = "beta-cluster-${random_string.suffix.result}" + project_id = var.project_ids[0] } resource "google_kms_key_ring" "db" { location = var.region name = "${local.name}-db" + project = local.project_id } resource "google_kms_crypto_key" "db" { @@ -45,7 +34,7 @@ module "this" { source = "../../../examples/simple_regional_beta" cluster_name_suffix = "-${random_string.suffix.result}" - project_id = var.project_id + project_id = local.project_id regional = false region = var.region zones = slice(var.zones, 0, 1) diff --git a/test/fixtures/beta_cluster/network.tf b/test/fixtures/beta_cluster/network.tf index 0a3f091958..c173435cbe 100644 --- a/test/fixtures/beta_cluster/network.tf +++ b/test/fixtures/beta_cluster/network.tf @@ -23,6 +23,7 @@ resource "random_string" "suffix" { resource "google_compute_network" "main" { name = "cft-gke-test-${random_string.suffix.result}" auto_create_subnetworks = false + project = local.project_id } resource "google_compute_subnetwork" "main" { @@ -30,6 +31,7 @@ resource "google_compute_subnetwork" "main" { ip_cidr_range = "10.0.0.0/17" region = var.region network = google_compute_network.main.self_link + project = local.project_id secondary_ip_range { range_name = "cft-gke-test-pods-${random_string.suffix.result}" @@ -41,4 +43,3 @@ resource "google_compute_subnetwork" "main" { ip_cidr_range = "192.168.64.0/18" } } - diff --git a/test/fixtures/beta_cluster/outputs.tf b/test/fixtures/beta_cluster/outputs.tf index da60cf87cf..f2d5730ec1 100644 --- a/test/fixtures/beta_cluster/outputs.tf +++ b/test/fixtures/beta_cluster/outputs.tf @@ -15,7 +15,7 @@ */ output "project_id" { - value = var.project_id + value = local.project_id } output "region" { diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 4bf09d5eb5..8685b9af5c 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -75,10 +75,10 @@ resource "google_service_account_key" "int_test" { } resource "google_project_iam_binding" "kubernetes_engine_kms_access" { - project = module.gke-project.project_id + project = module.gke-project-1.project_id role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" members = [ - "serviceAccount:service-${module.gke-project.project_number}@container-engine-robot.iam.gserviceaccount.com", + "serviceAccount:service-${module.gke-project-1.project_number}@container-engine-robot.iam.gserviceaccount.com", ] } From da4ad51f169d3faef7cffaa3a1c6ba3c8ad878ca Mon Sep 17 00:00:00 2001 From: Aaron Lane Date: Fri, 29 Nov 2019 12:47:33 -0500 Subject: [PATCH 077/110] Fix check for network policy config --- test/integration/beta_cluster/controls/gcloud.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/integration/beta_cluster/controls/gcloud.rb b/test/integration/beta_cluster/controls/gcloud.rb index 455a81cb61..032be9601a 100644 --- a/test/integration/beta_cluster/controls/gcloud.rb +++ b/test/integration/beta_cluster/controls/gcloud.rb @@ -55,9 +55,7 @@ "kubernetesDashboard" => { "disabled" => true, }, - "networkPolicyConfig" => { - "disabled" => true, - }, + "networkPolicyConfig" => {}, "istioConfig" => {}, "cloudRunConfig" => {}, }) From 7ec9c08a8a08a9c112c86ad18557cca33f07f7b9 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Thu, 14 Nov 2019 14:19:24 -0500 Subject: [PATCH 078/110] Add more detailed documentation for private cluster connectivity. --- autogen/README.md | 13 +++++++++++-- .../beta-private-cluster-update-variant/README.md | 11 ++++++++++- modules/beta-private-cluster/README.md | 11 ++++++++++- modules/private-cluster-update-variant/README.md | 11 ++++++++++- modules/private-cluster/README.md | 11 ++++++++++- 5 files changed, 51 insertions(+), 6 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index c8e956a76e..03179b90d7 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -12,9 +12,18 @@ The resources/services/activations/deletions that this module will create/trigge Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. {% if private_cluster %} -**Note**: You must run Terraform from a VM on the same VPC as your cluster, otherwise there will be issues connecting to the GKE master. +## Private Cluster Endpoints +When creating a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), nodes are provisioned with private IPs. +The Kubernetes master endpoint is also [locked down](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#access_to_the_cluster_endpoints), which affects these module features: +- `configure_ip_masq` +- `stub_domains` - {% endif %} +If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. +If you are using these features with a private cluster, you will need to either: +1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. +2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. + +{% endif %} ## Compatibility diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 3577bac930..c91b69bd62 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -10,7 +10,16 @@ The resources/services/activations/deletions that this module will create/trigge Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. -**Note**: You must run Terraform from a VM on the same VPC as your cluster, otherwise there will be issues connecting to the GKE master. +## Private Cluster Endpoints +When creating a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), nodes are provisioned with private IPs. +The Kubernetes master endpoint is also [locked down](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#access_to_the_cluster_endpoints), which affects these module features: +- `configure_ip_masq` +- `stub_domains` + +If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. +If you are using these features with a private cluster, you will need to either: +1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. +2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. ## Compatibility diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 8a799ccd3f..3a77c19984 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -10,7 +10,16 @@ The resources/services/activations/deletions that this module will create/trigge Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. -**Note**: You must run Terraform from a VM on the same VPC as your cluster, otherwise there will be issues connecting to the GKE master. +## Private Cluster Endpoints +When creating a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), nodes are provisioned with private IPs. +The Kubernetes master endpoint is also [locked down](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#access_to_the_cluster_endpoints), which affects these module features: +- `configure_ip_masq` +- `stub_domains` + +If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. +If you are using these features with a private cluster, you will need to either: +1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. +2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. ## Compatibility diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index e143917f25..c15361a377 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -10,7 +10,16 @@ The resources/services/activations/deletions that this module will create/trigge Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. -**Note**: You must run Terraform from a VM on the same VPC as your cluster, otherwise there will be issues connecting to the GKE master. +## Private Cluster Endpoints +When creating a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), nodes are provisioned with private IPs. +The Kubernetes master endpoint is also [locked down](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#access_to_the_cluster_endpoints), which affects these module features: +- `configure_ip_masq` +- `stub_domains` + +If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. +If you are using these features with a private cluster, you will need to either: +1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. +2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. ## Compatibility diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index eed365a5c4..aa4f0008ba 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -10,7 +10,16 @@ The resources/services/activations/deletions that this module will create/trigge Sub modules are provided from creating private clusters, beta private clusters, and beta public clusters as well. Beta sub modules allow for the use of various GKE beta features. See the modules directory for the various sub modules. -**Note**: You must run Terraform from a VM on the same VPC as your cluster, otherwise there will be issues connecting to the GKE master. +## Private Cluster Endpoints +When creating a [private cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters), nodes are provisioned with private IPs. +The Kubernetes master endpoint is also [locked down](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#access_to_the_cluster_endpoints), which affects these module features: +- `configure_ip_masq` +- `stub_domains` + +If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. +If you are using these features with a private cluster, you will need to either: +1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. +2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. ## Compatibility From 8ad1634ecf1afbb364c6bd2e0631fa0204f07f5c Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sat, 30 Nov 2019 20:54:50 -0500 Subject: [PATCH 079/110] Amend master authorized docs --- autogen/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/autogen/README.md b/autogen/README.md index 03179b90d7..da942c1ad6 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -21,7 +21,9 @@ The Kubernetes master endpoint is also [locked down](https://cloud.google.com/ku If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. If you are using these features with a private cluster, you will need to either: 1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. -2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. +2. Enable (beta) [route export functionality](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#master-on-prem-routing) to connect from an on-premise network over a VPN or Interconnect. +3. Include the external IP of your Terraform deployer in the `master_authorized_networks` configuration. Note that only IP addresses reserved in Google Cloud (such as in other VPCs) can be whitelisted. +4. Deploy a [bastion host](https://github.com/terraform-google-modules/terraform-google-bastion-host) or [proxy](https://cloud.google.com/solutions/creating-kubernetes-engine-private-clusters-with-net-proxies) in the same VPC as your GKE cluster. {% endif %} From ca6e16c40732a27d480823fee452f6ca3fec6db7 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sat, 30 Nov 2019 20:56:11 -0500 Subject: [PATCH 080/110] Regenerate docs --- modules/beta-private-cluster-update-variant/README.md | 4 +++- modules/beta-private-cluster/README.md | 4 +++- modules/private-cluster-update-variant/README.md | 4 +++- modules/private-cluster/README.md | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index c91b69bd62..2cfe1c9bf4 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -19,7 +19,9 @@ The Kubernetes master endpoint is also [locked down](https://cloud.google.com/ku If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. If you are using these features with a private cluster, you will need to either: 1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. -2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. +2. Enable (beta) [route export functionality](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#master-on-prem-routing) to connect from an on-premise network over a VPN or Interconnect. +3. Include the external IP of your Terraform deployer in the `master_authorized_networks` configuration. Note that only IP addresses reserved in Google Cloud (such as in other VPCs) can be whitelisted. +4. Deploy a [bastion host](https://github.com/terraform-google-modules/terraform-google-bastion-host) or [proxy](https://cloud.google.com/solutions/creating-kubernetes-engine-private-clusters-with-net-proxies) in the same VPC as your GKE cluster. ## Compatibility diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 3a77c19984..6917cb6007 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -19,7 +19,9 @@ The Kubernetes master endpoint is also [locked down](https://cloud.google.com/ku If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. If you are using these features with a private cluster, you will need to either: 1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. -2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. +2. Enable (beta) [route export functionality](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#master-on-prem-routing) to connect from an on-premise network over a VPN or Interconnect. +3. Include the external IP of your Terraform deployer in the `master_authorized_networks` configuration. Note that only IP addresses reserved in Google Cloud (such as in other VPCs) can be whitelisted. +4. Deploy a [bastion host](https://github.com/terraform-google-modules/terraform-google-bastion-host) or [proxy](https://cloud.google.com/solutions/creating-kubernetes-engine-private-clusters-with-net-proxies) in the same VPC as your GKE cluster. ## Compatibility diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index c15361a377..8215c1eeba 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -19,7 +19,9 @@ The Kubernetes master endpoint is also [locked down](https://cloud.google.com/ku If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. If you are using these features with a private cluster, you will need to either: 1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. -2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. +2. Enable (beta) [route export functionality](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#master-on-prem-routing) to connect from an on-premise network over a VPN or Interconnect. +3. Include the external IP of your Terraform deployer in the `master_authorized_networks` configuration. Note that only IP addresses reserved in Google Cloud (such as in other VPCs) can be whitelisted. +4. Deploy a [bastion host](https://github.com/terraform-google-modules/terraform-google-bastion-host) or [proxy](https://cloud.google.com/solutions/creating-kubernetes-engine-private-clusters-with-net-proxies) in the same VPC as your GKE cluster. ## Compatibility diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index aa4f0008ba..5f80738abf 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -19,7 +19,9 @@ The Kubernetes master endpoint is also [locked down](https://cloud.google.com/ku If you are *not* using these features, then the module will function normally for private clusters and no special configuration is needed. If you are using these features with a private cluster, you will need to either: 1. Run Terraform from a VM on the same VPC as your cluster (allowing it to connect to the private endpoint) and set `deploy_using_private_endpoint` to `true`. -2. Include the external IP of your Terraform deployer in the `master_authorized_networks_config`. +2. Enable (beta) [route export functionality](https://cloud.google.com/kubernetes-engine/docs/how-to/private-clusters#master-on-prem-routing) to connect from an on-premise network over a VPN or Interconnect. +3. Include the external IP of your Terraform deployer in the `master_authorized_networks` configuration. Note that only IP addresses reserved in Google Cloud (such as in other VPCs) can be whitelisted. +4. Deploy a [bastion host](https://github.com/terraform-google-modules/terraform-google-bastion-host) or [proxy](https://cloud.google.com/solutions/creating-kubernetes-engine-private-clusters-with-net-proxies) in the same VPC as your GKE cluster. ## Compatibility From 4ea758443af0d36656bac47c449d39b737213815 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Sat, 30 Nov 2019 21:06:41 -0500 Subject: [PATCH 081/110] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f60301859..eaaa41aed3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -262,6 +262,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 +[#138]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/138 [#354]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/354 [#350]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/350 [#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 From 11797c4029b9e39f8ea35d33e7c7f4bf24e661a8 Mon Sep 17 00:00:00 2001 From: pp Date: Fri, 22 Nov 2019 17:43:26 +0200 Subject: [PATCH 082/110] Add variable to allow a pre-generated SSH key to be passed to the ACM module * Fix #329 --- modules/acm/README.md | 1 + modules/acm/main.tf | 2 +- modules/acm/variables.tf | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/acm/README.md b/modules/acm/README.md index bef7eccd53..9960667669 100644 --- a/modules/acm/README.md +++ b/modules/acm/README.md @@ -53,6 +53,7 @@ By default, this module will attempt to download the ACM operator from Google di | operator\_path | Path to the operator yaml config. If unset, will download from GCS releases. | string | `"null"` | no | | policy\_dir | Subfolder containing configs in ACM Git repo | string | n/a | yes | | project\_id | The project in which the resource belongs. | string | n/a | yes | +| ssh\_auth\_key | Key for Git authentication. Overrides 'create_ssh_key' variable. Can be set using 'file(path/to/file)'-function. | string | `"null"` | no | | sync\_branch | ACM repo Git branch | string | `"master"` | no | | sync\_repo | ACM Git repo address | string | n/a | yes | diff --git a/modules/acm/main.tf b/modules/acm/main.tf index 3a78172ad6..69b5755337 100644 --- a/modules/acm/main.tf +++ b/modules/acm/main.tf @@ -18,7 +18,7 @@ locals { cluster_endpoint = "https://${var.cluster_endpoint}" token = data.google_client_config.default.access_token cluster_ca_certificate = data.google_container_cluster.primary.master_auth.0.cluster_ca_certificate - private_key = var.create_ssh_key ? tls_private_key.git_creds[0].private_key_pem : "" + private_key = var.create_ssh_key && var.ssh_auth_key == null ? tls_private_key.git_creds[0].private_key_pem : var.ssh_auth_key download_operator = var.operator_path == null ? true : false operator_path = local.download_operator ? "${path.module}/config-management-operator.yaml" : var.operator_path } diff --git a/modules/acm/variables.tf b/modules/acm/variables.tf index 513556364d..d56f20b073 100644 --- a/modules/acm/variables.tf +++ b/modules/acm/variables.tf @@ -62,6 +62,12 @@ variable "create_ssh_key" { default = true } +variable "ssh_auth_key" { + description = "Key for Git authentication. Overrides 'create_ssh_key' variable. Can be set using 'file(path/to/file)'-function." + type = string + default = null +} + variable "enable_policy_controller" { description = "Whether to enable the ACM Policy Controller on the cluster" type = bool From 509afe619ace442b59caf088732cb08791d4f136 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 2 Dec 2019 11:25:26 -0500 Subject: [PATCH 083/110] Relax provider version constraint --- autogen/versions.tf.tmpl | 4 ++-- modules/beta-private-cluster-update-variant/versions.tf | 2 +- modules/beta-private-cluster/versions.tf | 2 +- modules/beta-public-cluster/versions.tf | 2 +- modules/private-cluster-update-variant/versions.tf | 2 +- modules/private-cluster/versions.tf | 2 +- versions.tf | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/autogen/versions.tf.tmpl b/autogen/versions.tf.tmpl index 387a2e37c8..3024374262 100644 --- a/autogen/versions.tf.tmpl +++ b/autogen/versions.tf.tmpl @@ -19,9 +19,9 @@ terraform { required_providers { {% if beta_cluster %} - google-beta = "~> 2.18.0" + google-beta = "~> 2.18" {% else %} - google = "~> 2.18.0" + google = "~> 2.18" {% endif %} } } diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index 8e29303fa9..cdf71b53b4 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google-beta = "~> 2.18.0" + google-beta = "~> 2.18" } } diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index 8e29303fa9..cdf71b53b4 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google-beta = "~> 2.18.0" + google-beta = "~> 2.18" } } diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 8e29303fa9..cdf71b53b4 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google-beta = "~> 2.18.0" + google-beta = "~> 2.18" } } diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index e4544656fa..38e1d1bf9c 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google = "~> 2.18.0" + google = "~> 2.18" } } diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index e4544656fa..38e1d1bf9c 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google = "~> 2.18.0" + google = "~> 2.18" } } diff --git a/versions.tf b/versions.tf index e4544656fa..38e1d1bf9c 100644 --- a/versions.tf +++ b/versions.tf @@ -18,6 +18,6 @@ terraform { required_version = ">= 0.12" required_providers { - google = "~> 2.18.0" + google = "~> 2.18" } } From 45a8e33cecb60b174553fae378fbb50f6fd3cdcb Mon Sep 17 00:00:00 2001 From: Aaron Lane <10655063+aaron-lane@users.noreply.github.com> Date: Mon, 2 Dec 2019 13:10:49 -0500 Subject: [PATCH 084/110] Add #359, v6.0.1 to CHANGELOG --- CHANGELOG.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eaaa41aed3..1311e6e625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,12 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] +## [v6.0.1] - 2019-12-02 + +### Fixed + +- The required Google provider constraint has been relaxed to `~> 2.18`. [#359] + ## [v6.0.0] - 2019-11-28 v6.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v6.0.md). @@ -243,7 +249,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o * Initial release of module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.0...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.1...HEAD +[v6.0.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.0...v6.0.1 [v6.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.2.0...v6.0.0 [v5.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...v5.2.0 [v5.1.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...v5.1.1 @@ -262,7 +269,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 -[#138]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/138 +[#359]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/359 [#354]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/354 [#350]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/350 [#340]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/340 @@ -317,6 +324,7 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [#151]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/151 [#149]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/149 [#148]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/148 +[#138]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/138 [#136]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/138 [#136]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/136 [#132]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/132 From c3b8fcd10374b1fa6b4b01408a44a146670aa964 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 2 Dec 2019 17:17:38 -0500 Subject: [PATCH 085/110] Compute region output for zonal clusters --- autogen/main.tf.tmpl | 11 +++++------ main.tf | 11 +++++------ modules/beta-private-cluster-update-variant/main.tf | 11 +++++------ modules/beta-private-cluster/main.tf | 11 +++++------ modules/beta-public-cluster/main.tf | 11 +++++------ modules/private-cluster-update-variant/main.tf | 11 +++++------ modules/private-cluster/main.tf | 11 +++++------ 7 files changed, 35 insertions(+), 42 deletions(-) diff --git a/autogen/main.tf.tmpl b/autogen/main.tf.tmpl index 3ac28cc16a..a9dc61b013 100644 --- a/autogen/main.tf.tmpl +++ b/autogen/main.tf.tmpl @@ -96,8 +96,6 @@ locals { {% endif %} cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -137,11 +135,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/main.tf b/main.tf index 409cbfe55b..5cd5b596bf 100644 --- a/main.tf +++ b/main.tf @@ -64,8 +64,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -92,11 +90,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 9afc2502b8..1e126eca41 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -88,8 +88,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -123,11 +121,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 9afc2502b8..1e126eca41 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -88,8 +88,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -123,11 +121,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 5cff8bdd4e..b64c40cd15 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -88,8 +88,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -123,11 +121,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 7dea99f25c..c36ff21005 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -64,8 +64,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -92,11 +90,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 7dea99f25c..c36ff21005 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -64,8 +64,6 @@ locals { cluster_output_name = google_container_cluster.primary.name - cluster_output_location = google_container_cluster.primary.location - cluster_output_region = google_container_cluster.primary.region cluster_output_regional_zones = google_container_cluster.primary.node_locations cluster_output_zonal_zones = local.zone_count > 1 ? slice(var.zones, 1, local.zone_count) : [] cluster_output_zones = local.cluster_output_regional_zones @@ -92,11 +90,12 @@ locals { cluster_master_auth_list_layer1 = local.cluster_output_master_auth cluster_master_auth_list_layer2 = local.cluster_master_auth_list_layer1[0] cluster_master_auth_map = local.cluster_master_auth_list_layer2[0] - # cluster locals + + cluster_location = google_container_cluster.primary.location + cluster_region = var.regional ? google_container_cluster.primary.region : join("-", slice(split("-", local.cluster_location), 0, 2)) + cluster_zones = sort(local.cluster_output_zones) + cluster_name = local.cluster_output_name - cluster_location = local.cluster_output_location - cluster_region = local.cluster_output_region - cluster_zones = sort(local.cluster_output_zones) cluster_endpoint = local.cluster_output_endpoint cluster_ca_certificate = local.cluster_master_auth_map["cluster_ca_certificate"] cluster_master_version = local.cluster_output_master_version From 85f706d43aee9d590c3024fdefa43ead19ad04fd Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 2 Dec 2019 18:32:22 -0500 Subject: [PATCH 086/110] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1311e6e625..08b16665ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] +### Added +- Support for using a pre-existing Service Account with the ACM submodule. [#346](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/346) + ## [v6.0.1] - 2019-12-02 ### Fixed From 01712fa46c1144f9cd6f3f704cfd309ba8152550 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 2 Dec 2019 18:32:58 -0500 Subject: [PATCH 087/110] Fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08b16665ed..4af17aed2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ v6.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./d ### Removed * **Breaking**: Removed support for enabling the Kubernetes dashboard, as this is deprecated on GKE. [#337] -* **Beaking**: Removed support for versions of the Google provider and the Google Beta provider older than 2.18. [#261] +* **Breaking**: Removed support for versions of the Google provider and the Google Beta provider older than 2.18. [#261] * **Breaking**: Removed the `master_authorized_networks_config` variable. [#354] ### Fixed From 9132bbe77574f156b6dca221f7b374dc72f54712 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Mon, 2 Dec 2019 18:33:19 -0500 Subject: [PATCH 088/110] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af17aed2d..07d247cc99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ Extending the adopted spec, each change should have a link to its corresponding ### Fixed -- The required Google provider constraint has been relaxed to `~> 2.18`. [#359] +- The required Google provider constraint has been relaxed to `~> 2.18` (>= 2.18, <3.0). [#359] ## [v6.0.0] - 2019-11-28 From 5bc7419d191a84b350bf33e39b5c522091b6bad2 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 3 Dec 2019 10:57:21 -0500 Subject: [PATCH 089/110] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07d247cc99..d24679243c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,14 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] +## [v6.1.0] - 2019-12-03 + ### Added - Support for using a pre-existing Service Account with the ACM submodule. [#346](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/346) +### Fixed +- Compute region output for zonal clusters. [#362](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/362) + ## [v6.0.1] - 2019-12-02 ### Fixed From 228303291e4ff9fb48ac610327e2d1a92b4d3560 Mon Sep 17 00:00:00 2001 From: Morgante Pell Date: Tue, 3 Dec 2019 10:57:47 -0500 Subject: [PATCH 090/110] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d24679243c..00f9d7ce0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -257,7 +257,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o * Initial release of module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.1...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.1.0...HEAD +[v6.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.1...v6.1.0 [v6.0.1]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v6.0.0...v6.0.1 [v6.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.2.0...v6.0.0 [v5.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.1...v5.2.0 From 97cde1999107e987f6afd3e70a56c16b9becc5a9 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Tue, 3 Dec 2019 17:50:46 -0600 Subject: [PATCH 091/110] Added beta clusters conditionals --- autogen/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/autogen/README.md b/autogen/README.md index 40c33bf7c3..2d368345ee 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -197,23 +197,35 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +{% if beta_cluster %} | effect | Effect for the taint | | Required | +{% endif %} | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| key | The key required for the taint | | Required | +{% if beta_cluster %} +| key | The key required for the taint | | Required | +{% endif %} | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +{% if beta_cluster %} | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | +{% endif %} | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1 | | Required (when autoscaling is false) | +{% if beta_cluster %} | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Required | +{% endif %} | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +{% if beta_cluster %} | sandbox_type | Sandbox to use for pods in the node pool | | Required | +{% endif %} | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | +{% if beta_cluster %} | value | The value for the taint | | Required | +{% endif %} ## File structure The project has the following folders and files: From 7b10db3e08329782fb48e91073fc347b1a252589 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Thu, 24 Oct 2019 13:29:20 -0500 Subject: [PATCH 092/110] Updated node_pools variable documentation --- autogen/README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/autogen/README.md b/autogen/README.md index d7dc65c1b9..9d032ec773 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -180,6 +180,22 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools Variable +The node_pools variable takes the following parameters: + +name - (Optional) The name of the node pool. If left blank, Terraform will auto-generate a unique name +min_count - (Required) Minimum number of nodes in the NodePool. Must be >=0 and <= max_count +max_count - (Required) Maximum number of nodes in the NodePool. Must be >= min_count +machine_type - (Optional) The name of a Google Compute Engine machine type. Defaults to n1-standard-1 +disk_size_gb - (Optional) Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB +disk_type - (Optional) Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' +image_type - (Optional) The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool +preemptible - (Optional) A boolean that represents whether or not the underlying node VMs are preemptible +service_account - (Optional) The service account to be used by the Node VMs. If not specified, the "default" service account is used +initial_node_count - (Optional) The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource +auto_repair - (Optional) Whether the nodes will be automatically repaired +auto_upgrade - (Optional) Whether the nodes will be automatically upgraded + ## File structure The project has the following folders and files: From 515e5520adc5fff11639d844f042124f62abf6e1 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Thu, 24 Oct 2019 16:02:32 -0500 Subject: [PATCH 093/110] Added tables to the documentation --- autogen/README.md | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 9d032ec773..14a47df867 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -183,18 +183,20 @@ In order to operate with the Service Account you must activate the following API ## node_pools Variable The node_pools variable takes the following parameters: -name - (Optional) The name of the node pool. If left blank, Terraform will auto-generate a unique name -min_count - (Required) Minimum number of nodes in the NodePool. Must be >=0 and <= max_count -max_count - (Required) Maximum number of nodes in the NodePool. Must be >= min_count -machine_type - (Optional) The name of a Google Compute Engine machine type. Defaults to n1-standard-1 -disk_size_gb - (Optional) Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB -disk_type - (Optional) Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' -image_type - (Optional) The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool -preemptible - (Optional) A boolean that represents whether or not the underlying node VMs are preemptible -service_account - (Optional) The service account to be used by the Node VMs. If not specified, the "default" service account is used -initial_node_count - (Optional) The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource -auto_repair - (Optional) Whether the nodes will be automatically repaired -auto_upgrade - (Optional) Whether the nodes will be automatically upgraded +| Name | Description | Requirement | +| --- | --- | --- | +| name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | Required | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | Required | +| machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | Optional | +| service_account | The service account to be used by the Node VMs. If not specified, the "default" service account is used | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | Optional | +| auto_repair | Whether the nodes will be automatically repaired | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | Optional | ## File structure The project has the following folders and files: From 2db2c4d3b2bf39a1ebea17b106351d6db1a1ee61 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Wed, 30 Oct 2019 17:44:28 -0500 Subject: [PATCH 094/110] Made changes to the variables --- autogen/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 14a47df867..77a7843297 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -180,14 +180,14 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools Variable +## node_pools variable The node_pools variable takes the following parameters: | Name | Description | Requirement | | --- | --- | --- | | name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | Required | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | Required | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | | machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | From 6d428e72b05b55b70f90465d11f07fb6eb5425e5 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 10:55:09 -0500 Subject: [PATCH 095/110] Added the autoscaling property --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index 77a7843297..ccbca57042 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -186,6 +186,7 @@ The node_pools variable takes the following parameters: | Name | Description | Requirement | | --- | --- | --- | | name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage. Default value is true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | | machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | From afbc5ecc238f98af2f290070a0376441193cf998 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 12:48:31 -0500 Subject: [PATCH 096/110] Default column added --- autogen/README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index ccbca57042..8e980e7c7e 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -183,21 +183,21 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Requirement | -| --- | --- | --- | -| name | The name of the node pool. If left blank, Terraform will auto-generate a unique name | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage. Default value is true | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Default value is 1 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Default value is 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type. Defaults to n1-standard-1 | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB. Defaults to 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd'). If unspecified, the default disk type is 'pd-standard' | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | Optional | -| service_account | The service account to be used by the Node VMs. If not specified, the "default" service account is used | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | Optional | -| auto_repair | Whether the nodes will be automatically repaired | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | Optional | +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| name | The name of the node pool | "" | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-1 | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | "" | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | ## File structure The project has the following folders and files: From 9df34fb7d14c0c585636edf95dbd8d8b525cb9e4 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 1 Nov 2019 12:50:29 -0500 Subject: [PATCH 097/110] Default column added --- autogen/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 8e980e7c7e..7633c56454 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -185,7 +185,7 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | -| name | The name of the node pool | "" | Optional | +| name | The name of the node pool | " " | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | @@ -194,7 +194,7 @@ The node_pools variable takes the following parameters: | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | "" | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | From 93e8e7c80c74394c732c7726b2f68ac991290828 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 4 Nov 2019 15:18:05 -0600 Subject: [PATCH 098/110] Made changes to the table --- autogen/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 7633c56454..bf02a861bd 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -189,7 +189,7 @@ The node_pools variable takes the following parameters: | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-1 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | @@ -197,7 +197,7 @@ The node_pools variable takes the following parameters: | service_account | The service account to be used by the Node VMs | " " | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | false | Optional | ## File structure The project has the following folders and files: From 62963c15d46204c0ab97dc6bbf7ec5463919e179 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 4 Nov 2019 23:13:27 -0600 Subject: [PATCH 099/110] Added node_locations --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index bf02a861bd..701f499f3d 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -187,6 +187,7 @@ The node_pools variable takes the following parameters: | --- | --- | --- | --- | | name | The name of the node pool | " " | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | From 24cad172c66855d82255b3b87784252a3ff8d16f Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Fri, 8 Nov 2019 23:41:26 -0600 Subject: [PATCH 100/110] Updated auto upgrade --- autogen/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index 701f499f3d..39fba985d6 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -185,20 +185,20 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | -| name | The name of the node pool | " " | Optional | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | false | Optional | ## File structure The project has the following folders and files: From 363d850a0b43d14b9b473d72f44ed8643909e387 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 10:43:50 -0600 Subject: [PATCH 101/110] node count added --- autogen/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/autogen/README.md b/autogen/README.md index 39fba985d6..abedc4b322 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -196,6 +196,7 @@ The node_pools variable takes the following parameters: | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | | name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | | service_account | The service account to be used by the Node VMs | " " | Optional | From 6e609f13aa9391250b96982096cf5fc894d7d1a8 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:39:17 -0600 Subject: [PATCH 102/110] Fixed Errors --- README.md | 24 +++++++++++++++++++ .../README.md | 21 ++++++++++++++++ modules/beta-private-cluster/README.md | 21 ++++++++++++++++ modules/beta-public-cluster/README.md | 21 ++++++++++++++++ .../private-cluster-update-variant/README.md | 21 ++++++++++++++++ modules/private-cluster/README.md | 21 ++++++++++++++++ 6 files changed, 129 insertions(+) diff --git a/README.md b/README.md index 9e0bdc76d8..e3af1a0aa4 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,7 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure +<<<<<<< HEAD ## Inputs | Name | Description | Type | Default | Required | @@ -182,6 +183,8 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | +======= +>>>>>>> Fixed Errors ## Requirements @@ -221,6 +224,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 8bc9626de1..a55ef21373 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -268,6 +268,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 96ff7b416b..b49ccfbcbb 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -268,6 +268,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fbc72a36f..0846b0e1af 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -248,6 +248,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 8e5c574893..dc2c7ec2f2 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -241,6 +241,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3efa415a4a..2dd19a9001 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -241,6 +241,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: From 73f788af9a981d7f003b3b838c35b9dd1ed2c4a5 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:42:44 -0600 Subject: [PATCH 103/110] Fixed Errors --- README.md | 26 ++++++++++++++++++++++++++ modules/beta-private-cluster/README.md | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/README.md b/README.md index e3af1a0aa4..262d55ef8d 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,9 @@ Then perform the following commands on the root folder: <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> Fixed Errors ## Inputs | Name | Description | Type | Default | Required | @@ -131,6 +134,7 @@ Then perform the following commands on the root folder: | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | +<<<<<<< HEAD | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | @@ -139,6 +143,17 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | | network\_policy | Enable network policy addon | bool | `"true"` | no | +======= +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | +| name | The name of the cluster (required) | string | n/a | yes | +| network | The VPC network to host the cluster in (required) | string | n/a | yes | +| network\_policy | Enable network policy addon | bool | `"false"` | no | +>>>>>>> Fixed Errors | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | @@ -157,7 +172,11 @@ Then perform the following commands on the root folder: | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +<<<<<<< HEAD | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | +======= +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +>>>>>>> Fixed Errors | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs @@ -168,6 +187,10 @@ Then perform the following commands on the root folder: | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | +<<<<<<< HEAD +======= +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +>>>>>>> Fixed Errors | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -183,6 +206,9 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | +<<<<<<< HEAD +======= +>>>>>>> Fixed Errors ======= >>>>>>> Fixed Errors diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index b49ccfbcbb..17b6952344 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -136,7 +136,10 @@ Then perform the following commands on the root folder: | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | +<<<<<<< HEAD | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | +======= +>>>>>>> Fixed Errors | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | @@ -163,15 +166,27 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | | istio | (Beta) Enable Istio addon | string | `"false"` | no | +<<<<<<< HEAD | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +======= +| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | +| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | +| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | +| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | +| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | +>>>>>>> Fixed Errors | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | +<<<<<<< HEAD | network\_policy | Enable network policy addon | bool | `"true"` | no | +======= +| network\_policy | Enable network policy addon | bool | `"false"` | no | +>>>>>>> Fixed Errors | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | @@ -196,7 +211,11 @@ Then perform the following commands on the root folder: | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | +<<<<<<< HEAD | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | +======= +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +>>>>>>> Fixed Errors | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs @@ -211,6 +230,10 @@ Then perform the following commands on the root folder: | identity\_namespace | Workload Identity namespace | | intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | istio\_enabled | Whether Istio is enabled | +<<<<<<< HEAD +======= +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +>>>>>>> Fixed Errors | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | From 34838b0eee635390ffd6d3d3a6375d003db77a56 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:48:46 -0600 Subject: [PATCH 104/110] Fixed Errors --- README.md | 2 +- autogen/README.md | 2 +- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-public-cluster/README.md | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 262d55ef8d..48e9dff19b 100644 --- a/README.md +++ b/README.md @@ -253,7 +253,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/autogen/README.md b/autogen/README.md index abedc4b322..e03d494a6b 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -183,7 +183,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index a55ef21373..24a3cf009e 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -271,7 +271,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 17b6952344..59733965f8 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -294,7 +294,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 0846b0e1af..215447f7df 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -251,7 +251,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index dc2c7ec2f2..635bf095a1 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -244,7 +244,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 2dd19a9001..38735022f3 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -244,7 +244,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | From c2e3b04ebe221ccf538468d90ad093111ada7212 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Wed, 20 Nov 2019 14:05:02 -0600 Subject: [PATCH 105/110] Changes to the variables --- autogen/README.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/autogen/README.md b/autogen/README.md index e03d494a6b..3536bf7a36 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -185,21 +185,30 @@ The node_pools variable takes the following parameters: | Name | Description | Default | Requirement | | --- | --- | --- | --- | +| accelerator_count | The number of the guest accelerator cards exposed to this instance | 0 | Optional | +| accelerator_type | The accelerator type resource to expose to the instance | " " | Optional | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| effect | Effect for the taint | | Required | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| key | The key required for the taint | | Required | | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | +| name | The name of the node pool | | Required | +| node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1 | | Required (when autoscaling is false) | | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| node_metadata | Options to expose the node metadata to the workload running on the node | | Required | | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| sandbox_type | Sandbox to use for pods in the node pool | | Required | | service_account | The service account to be used by the Node VMs | " " | Optional | +| tags | The list of instance tags applied to all nodes | | Required | +| value | The value for the taint | | Required | ## File structure The project has the following folders and files: From e1bf73e80e1cc4898278dc88fe3208e4e5105291 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Tue, 3 Dec 2019 17:50:46 -0600 Subject: [PATCH 106/110] Added beta clusters conditionals --- autogen/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/autogen/README.md b/autogen/README.md index 3536bf7a36..f38fb6a29f 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -192,23 +192,35 @@ The node_pools variable takes the following parameters: | auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | | disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | | disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +{% if beta_cluster %} | effect | Effect for the taint | | Required | +{% endif %} | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| key | The key required for the taint | | Required | +{% if beta_cluster %} +| key | The key required for the taint | | Required | +{% endif %} | machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +{% if beta_cluster %} | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | +{% endif %} | min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count. Should be used when autoscaling is true | 1 | Optional | | name | The name of the node pool | | Required | | node_count | The number of nodes in the nodepool when autoscaling is false. Otherwise defaults to 1 | | Required (when autoscaling is false) | +{% if beta_cluster %} | node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | | node_metadata | Options to expose the node metadata to the workload running on the node | | Required | +{% endif %} | preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +{% if beta_cluster %} | sandbox_type | Sandbox to use for pods in the node pool | | Required | +{% endif %} | service_account | The service account to be used by the Node VMs | " " | Optional | | tags | The list of instance tags applied to all nodes | | Required | +{% if beta_cluster %} | value | The value for the taint | | Required | +{% endif %} ## File structure The project has the following folders and files: From 692450d4d57d360e12c55add42adc40c43eafd0c Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Tue, 3 Dec 2019 19:22:55 -0600 Subject: [PATCH 107/110] Removing Conflicts --- .../README.md | 21 ------------------- modules/beta-private-cluster/README.md | 21 ------------------- modules/beta-public-cluster/README.md | 21 ------------------- .../private-cluster-update-variant/README.md | 21 ------------------- modules/private-cluster/README.md | 21 ------------------- 5 files changed, 105 deletions(-) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 24a3cf009e..8bc9626de1 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -268,27 +268,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools variable -The node_pools variable takes the following parameters: - -| Name | Description | Default | Requirement | -| --- | --- | --- | --- | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | " " | Optional | - ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 59733965f8..9c4af26b8e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -291,27 +291,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools variable -The node_pools variable takes the following parameters: - -| Name | Description | Default | Requirement | -| --- | --- | --- | --- | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | " " | Optional | - ## File structure The project has the following folders and files: diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 215447f7df..7fbc72a36f 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -248,27 +248,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools variable -The node_pools variable takes the following parameters: - -| Name | Description | Default | Requirement | -| --- | --- | --- | --- | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | " " | Optional | - ## File structure The project has the following folders and files: diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 635bf095a1..8e5c574893 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -241,27 +241,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools variable -The node_pools variable takes the following parameters: - -| Name | Description | Default | Requirement | -| --- | --- | --- | --- | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | " " | Optional | - ## File structure The project has the following folders and files: diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 38735022f3..3efa415a4a 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -241,27 +241,6 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com -## node_pools variable -The node_pools variable takes the following parameters: - -| Name | Description | Default | Requirement | -| --- | --- | --- | --- | -| auto_repair | Whether the nodes will be automatically repaired | true | Optional | -| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | -| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | -| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | -| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | -| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | -| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | -| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | -| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | -| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | -| name | The name of the node pool | " " | Optional | -| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | -| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | -| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | -| service_account | The service account to be used by the Node VMs | " " | Optional | - ## File structure The project has the following folders and files: From bb8da13694c07ab7b38bb656a8e1dc112b8579c6 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:39:17 -0600 Subject: [PATCH 108/110] Fixed Errors --- README.md | 7 +++ .../README.md | 21 +++++++++ modules/beta-private-cluster/README.md | 45 +++++++++---------- modules/beta-public-cluster/README.md | 22 ++++++++- .../private-cluster-update-variant/README.md | 21 +++++++++ modules/private-cluster/README.md | 21 +++++++++ 6 files changed, 112 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 48e9dff19b..078b301a8c 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ Then perform the following commands on the root folder: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> Fixed Errors ## Inputs @@ -211,6 +212,8 @@ Then perform the following commands on the root folder: >>>>>>> Fixed Errors ======= >>>>>>> Fixed Errors +======= +>>>>>>> b18a5df... Fixed Errors ## Requirements @@ -253,7 +256,11 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: +<<<<<<< HEAD | Name | Description | Default | Requirement | +======= +| Name | Description | Default | Requirement | +>>>>>>> b18a5df... Fixed Errors | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 8bc9626de1..a55ef21373 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -268,6 +268,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 9c4af26b8e..a76bd5d7c5 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -136,10 +136,7 @@ Then perform the following commands on the root folder: | basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | | basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | | cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -<<<<<<< HEAD | cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | object | `` | no | -======= ->>>>>>> Fixed Errors | cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | | cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | | configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | @@ -166,27 +163,15 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | | istio | (Beta) Enable Istio addon | string | `"false"` | no | -<<<<<<< HEAD | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -======= -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | ->>>>>>> Fixed Errors | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | -<<<<<<< HEAD | network\_policy | Enable network policy addon | bool | `"true"` | no | -======= -| network\_policy | Enable network policy addon | bool | `"false"` | no | ->>>>>>> Fixed Errors | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | @@ -211,11 +196,7 @@ Then perform the following commands on the root folder: | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -<<<<<<< HEAD | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | -======= -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | ->>>>>>> Fixed Errors | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs @@ -230,10 +211,6 @@ Then perform the following commands on the root folder: | identity\_namespace | Workload Identity namespace | | intranode\_visibility\_enabled | Whether intra-node visibility is enabled | | istio\_enabled | Whether Istio is enabled | -<<<<<<< HEAD -======= -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | ->>>>>>> Fixed Errors | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -251,7 +228,6 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | | zones | List of zones in which the cluster resides | - ## Requirements @@ -291,6 +267,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7fbc72a36f..7d1a203576 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -208,7 +208,6 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | | zones | List of zones in which the cluster resides | - ## Requirements @@ -248,6 +247,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 8e5c574893..dc2c7ec2f2 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -241,6 +241,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3efa415a4a..2dd19a9001 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -241,6 +241,27 @@ In order to operate with the Service Account you must activate the following API - Compute Engine API - compute.googleapis.com - Kubernetes Engine API - container.googleapis.com +## node_pools variable +The node_pools variable takes the following parameters: + +| Name | Description | Default | Requirement | +| --- | --- | --- | --- | +| auto_repair | Whether the nodes will be automatically repaired | true | Optional | +| autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | +| auto_upgrade | Whether the nodes will be automatically upgraded | true (if cluster is regional) | Optional | +| disk_size_gb | Size of the disk attached to each node, specified in GB. The smallest allowed disk size is 10GB | 100GB | Optional | +| disk_type | Type of the disk attached to each node (e.g. 'pd-standard' or 'pd-ssd') | pd-standard | Optional | +| image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS | Optional | +| initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource | 0 | Optional | +| machine_type | The name of a Google Compute Engine machine type | n1-standard-2 | Optional | +| max_count | Maximum number of nodes in the NodePool. Must be >= min_count | 100 | Optional | +| min_count | Minimum number of nodes in the NodePool. Must be >=0 and <= max_count | 1 | Optional | +| name | The name of the node pool | " " | Optional | +| node_count | The number of nodes in the nodepool when autoscaling is false | 2 | Required (when autoscaling is false) | +| node_locations | The list of zones in which the cluster's nodes are located. Nodes must be in the region of their regional cluster or in the same region as their cluster's zone for zonal clusters | " " | Optional | +| preemptible | A boolean that represents whether or not the underlying node VMs are preemptible | false | Optional | +| service_account | The service account to be used by the Node VMs | " " | Optional | + ## File structure The project has the following folders and files: From c842a0912a0f6426bba839a856109c700a4271e1 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Wed, 4 Dec 2019 13:10:21 -0600 Subject: [PATCH 109/110] Resolved merge conflicts --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 078b301a8c..42ed04bbb3 100644 --- a/README.md +++ b/README.md @@ -112,8 +112,11 @@ Then perform the following commands on the root folder: <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> Fixed Errors +======= +>>>>>>> 2ae6c27... Fixed Errors ## Inputs | Name | Description | Type | Default | Required | @@ -136,6 +139,7 @@ Then perform the following commands on the root folder: | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | <<<<<<< HEAD +<<<<<<< HEAD | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | @@ -145,6 +149,8 @@ Then perform the following commands on the root folder: | network | The VPC network to host the cluster in (required) | string | n/a | yes | | network\_policy | Enable network policy addon | bool | `"true"` | no | ======= +======= +>>>>>>> 2ae6c27... Fixed Errors | kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | @@ -154,7 +160,10 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | | network\_policy | Enable network policy addon | bool | `"false"` | no | +<<<<<<< HEAD >>>>>>> Fixed Errors +======= +>>>>>>> 2ae6c27... Fixed Errors | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | @@ -174,10 +183,14 @@ Then perform the following commands on the root folder: | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | <<<<<<< HEAD +<<<<<<< HEAD | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | ======= | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | >>>>>>> Fixed Errors +======= +| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | +>>>>>>> 2ae6c27... Fixed Errors | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs @@ -189,9 +202,13 @@ Then perform the following commands on the root folder: | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | <<<<<<< HEAD +<<<<<<< HEAD ======= | kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | >>>>>>> Fixed Errors +======= +| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | +>>>>>>> 2ae6c27... Fixed Errors | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -207,6 +224,7 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> Fixed Errors @@ -214,6 +232,8 @@ Then perform the following commands on the root folder: >>>>>>> Fixed Errors ======= >>>>>>> b18a5df... Fixed Errors +======= +>>>>>>> 2ae6c27... Fixed Errors ## Requirements From 6dc75a238b2c322cace2044d6cd886118e72ee31 Mon Sep 17 00:00:00 2001 From: neelesh9795 Date: Mon, 18 Nov 2019 17:48:46 -0600 Subject: [PATCH 110/110] Fixed Errors --- README.md | 57 +------------------ .../README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-public-cluster/README.md | 2 +- .../private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 6 files changed, 6 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 42ed04bbb3..ee1c577d33 100644 --- a/README.md +++ b/README.md @@ -109,14 +109,7 @@ Then perform the following commands on the root folder: - `terraform destroy` to destroy the built infrastructure -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> Fixed Errors -======= ->>>>>>> 2ae6c27... Fixed Errors + ## Inputs | Name | Description | Type | Default | Required | @@ -138,8 +131,6 @@ Then perform the following commands on the root folder: | ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | | ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | | issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -<<<<<<< HEAD -<<<<<<< HEAD | kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | | logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | | maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | @@ -148,22 +139,6 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | string | n/a | yes | | network | The VPC network to host the cluster in (required) | string | n/a | yes | | network\_policy | Enable network policy addon | bool | `"true"` | no | -======= -======= ->>>>>>> 2ae6c27... Fixed Errors -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -<<<<<<< HEAD ->>>>>>> Fixed Errors -======= ->>>>>>> 2ae6c27... Fixed Errors | network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | @@ -182,15 +157,7 @@ Then perform the following commands on the root folder: | skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -<<<<<<< HEAD -<<<<<<< HEAD | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list(string) | `` | no | -======= -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | ->>>>>>> Fixed Errors -======= -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | ->>>>>>> 2ae6c27... Fixed Errors | zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs @@ -201,14 +168,6 @@ Then perform the following commands on the root folder: | endpoint | Cluster endpoint | | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | | http\_load\_balancing\_enabled | Whether http load balancing enabled | -<<<<<<< HEAD -<<<<<<< HEAD -======= -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | ->>>>>>> Fixed Errors -======= -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | ->>>>>>> 2ae6c27... Fixed Errors | location | Cluster location (region if regional cluster, zone if zonal cluster) | | logging\_service | Logging service used | | master\_authorized\_networks\_config | Networks from which access to master is permitted | @@ -224,16 +183,6 @@ Then perform the following commands on the root folder: | type | Cluster type (regional / zonal) | | zones | List of zones in which the cluster resides | -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> Fixed Errors -======= ->>>>>>> Fixed Errors -======= ->>>>>>> b18a5df... Fixed Errors -======= ->>>>>>> 2ae6c27... Fixed Errors ## Requirements @@ -276,11 +225,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -<<<<<<< HEAD | Name | Description | Default | Requirement | -======= -| Name | Description | Default | Requirement | ->>>>>>> b18a5df... Fixed Errors | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index a55ef21373..24a3cf009e 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -271,7 +271,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index a76bd5d7c5..d24f98c3ed 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -270,7 +270,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 7d1a203576..0175b877b4 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -250,7 +250,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index dc2c7ec2f2..635bf095a1 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -244,7 +244,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 2dd19a9001..38735022f3 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -244,7 +244,7 @@ In order to operate with the Service Account you must activate the following API ## node_pools variable The node_pools variable takes the following parameters: -| Name | Description | Default | Requirement | +| Name | Description | Default | Requirement | | --- | --- | --- | --- | | auto_repair | Whether the nodes will be automatically repaired | true | Optional | | autoscaling | Configuration required by cluster autoscaler to adjust the size of the node pool to the current cluster usage | true | Optional |