diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c645b5daf..5e609c0eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,22 @@ Extending the adopted spec, each change should have a link to its corresponding ## [Unreleased] -### Changed - -* Made `region` variable optional for zonal clusters [#247] +## [v5.1.0] - 2019-10-24 ### Added +* Added ability to skip local-exec provisioners. [#258] * Added [private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/private-cluster-update-variant) and [beta private](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/tree/master/modules/beta-private-cluster-update-variant) variants which allow node pools to be created before being destroyed. [#256] +* Add a parameter `registry_project_id` to allow connecting to registries in other projects. [#273] + +### Changed + +* Made `region` variable optional for zonal clusters. [#247] +* Made default metadata, labels, and tags optional. [#282] + +### Fixed + +* Authenticate gcloud in wait-for-cluster.sh using value of `GOOGLE_APPLICATION_CREDENTIALS`. [#284] [#285] ## [v5.0.0] - 2019-09-25 v5.0.0 is a backwards-incompatible release. Please see the [upgrading guide](./docs/upgrading_to_v5.0.md). @@ -189,7 +198,8 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o * Initial release of module. -[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...HEAD +[Unreleased]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.1.0...HEAD +[v5.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v5.0.0...v5.1.0 [v5.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.1.0...v5.0.0 [v4.1.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v4.0.0...v4.1.0 [v4.0.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v3.0.0...v4.0.0 @@ -204,9 +214,14 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o [v0.3.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.2.0...v0.3.0 [v0.2.0]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v0.1.0...v0.2.0 -[#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247 +[#285]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/285 +[#284]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/284 +[#282]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/282 +[#273]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/273 +[#258]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/258 [#256]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/256 [#248]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/248 +[#247]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/247 [#228]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/228 [#238]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/238 [#241]: https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/pull/241 diff --git a/Makefile b/Makefile index 5039822a75..52c9deaf8f 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.1.0 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.4.5 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd diff --git a/README.md b/README.md index 77967e84e0..15f6aff13b 100644 --- a/README.md +++ b/README.md @@ -167,8 +167,10 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -228,6 +230,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/autogen/README.md b/autogen/README.md index 421e4a2605..846d339911 100644 --- a/autogen/README.md +++ b/autogen/README.md @@ -139,102 +139,6 @@ Version 1.0.0 of this module introduces a breaking change: adding the `disable-l In either case, upgrading to module version `v1.0.0` will trigger a recreation of all node pools in the cluster. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|:----:|:-----:|:-----:| -| authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | string | `"null"` | no | -| basic\_auth\_password | The password to be used with Basic Authentication. | string | `""` | no | -| basic\_auth\_username | The username to be used with Basic Authentication. An empty value will disable Basic Authentication, which is the recommended configuration. | string | `""` | no | -| cloudrun | (Beta) Enable CloudRun addon | string | `"false"` | no | -| cluster\_ipv4\_cidr | The IP address range of the kubernetes pods in this cluster. Default is an automatically assigned CIDR. | string | `""` | no | -| cluster\_resource\_labels | The GCE resource labels (a map of key/value pairs) to be applied to the cluster | map(string) | `` | no | -| configure\_ip\_masq | Enables the installation of ip masquerading, which is usually no longer required when using aliasied IP addresses. IP masquerading uses a kubectl call, so when you have a private cluster, you will need access to the API server. | string | `"false"` | no | -| create\_service\_account | Defines if service account specified to run nodes should be created. | bool | `"true"` | no | -| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | -| default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | string | `"110"` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | bool | `"false"` | no | -| description | The description of the cluster | string | `""` | no | -| disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | bool | `"true"` | no | -| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | -| enable\_intranode\_visibility | Whether Intra-node visibility is enabled for this cluster. This makes same node pod to pod traffic visible for VPC network | bool | `"false"` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | bool | `"false"` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | bool | `"false"` | no | -| enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | bool | `"false"` | no | -| grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer role. | bool | `"false"` | no | -| horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | bool | `"true"` | no | -| http\_load\_balancing | Enable httpload balancer addon | bool | `"true"` | no | -| identity\_namespace | Workload Identity namespace | string | `""` | no | -| initial\_node\_count | The number of nodes to create in this cluster's default node pool. | number | `"0"` | no | -| ip\_masq\_link\_local | Whether to masquerade traffic to the link-local prefix (169.254.0.0/16). | bool | `"false"` | no | -| ip\_masq\_resync\_interval | The interval at which the agent attempts to sync its ConfigMap file from the disk. | string | `"60s"` | no | -| ip\_range\_pods | The _name_ of the secondary subnet ip range to use for pods | string | n/a | yes | -| ip\_range\_services | The _name_ of the secondary subnet range to use for services | string | n/a | yes | -| issue\_client\_certificate | Issues a client certificate to authenticate to the cluster endpoint. To maximize the security of your cluster, leave this option disabled. Client certificates don't automatically rotate and aren't easily revocable. WARNING: changing this after cluster creation is destructive! | bool | `"false"` | no | -| istio | (Beta) Enable Istio addon | string | `"false"` | no | -| kubernetes\_dashboard | Enable kubernetes dashboard addon | bool | `"false"` | no | -| kubernetes\_version | The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region. | string | `"latest"` | no | -| logging\_service | The logging service that the cluster should write logs to. Available options include logging.googleapis.com, logging.googleapis.com/kubernetes (beta), and none | string | `"logging.googleapis.com"` | no | -| maintenance\_start\_time | Time window specified for daily maintenance operations in RFC3339 format | string | `"05:00"` | no | -| master\_authorized\_networks\_config | The desired configuration options for master authorized networks. The object format is {cidr_blocks = list(object({cidr_block = string, display_name = string}))}. Omit the nested cidr_blocks attribute to disallow external access (except the cluster node IPs, which GKE automatically whitelists). | object | `` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | string | `"10.0.0.0/28"` | no | -| monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | string | `"monitoring.googleapis.com"` | no | -| name | The name of the cluster (required) | string | n/a | yes | -| network | The VPC network to host the cluster in (required) | string | n/a | yes | -| network\_policy | Enable network policy addon | bool | `"false"` | no | -| network\_policy\_provider | The network policy provider. | string | `"CALICO"` | no | -| network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | string | `""` | no | -| node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | -| node\_pools | List of maps containing node pools | list(map(string)) | `` | no | -| node\_pools\_labels | Map of maps containing node labels by node-pool name | map(map(string)) | `` | no | -| node\_pools\_metadata | Map of maps containing node metadata by node-pool name | map(map(string)) | `` | no | -| node\_pools\_oauth\_scopes | Map of lists containing node oauth scopes by node-pool name | map(list(string)) | `` | no | -| node\_pools\_tags | Map of lists containing node network tags by node-pool name | map(list(string)) | `` | no | -| node\_pools\_taints | Map of lists containing node taints by node-pool name | object | `` | no | -| node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | -| non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | -| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | -| project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | -| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | -| remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | -| resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | -| sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | -| service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | -| stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | -| subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | -| upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | -| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| ca\_certificate | Cluster ca certificate (base64 encoded) | -| cloudrun\_enabled | Whether CloudRun enabled | -| endpoint | Cluster endpoint | -| horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled | -| http\_load\_balancing\_enabled | Whether http load balancing enabled | -| intranode\_visibility\_enabled | Whether intra-node visibility is enabled | -| istio\_enabled | Whether Istio is enabled | -| kubernetes\_dashboard\_enabled | Whether kubernetes dashboard enabled | -| location | Cluster location (region if regional cluster, zone if zonal cluster) | -| logging\_service | Logging service used | -| master\_authorized\_networks\_config | Networks from which access to master is permitted | -| master\_version | Current master kubernetes version | -| min\_master\_version | Minimum master kubernetes version | -| monitoring\_service | Monitoring service used | -| name | Cluster name | -| network\_policy\_enabled | Whether network policy enabled | -| node\_pools\_names | List of node pools names | -| node\_pools\_versions | List of node pools versions | -| pod\_security\_policy\_enabled | Whether pod security policy is enabled | -| region | Cluster region | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | -| type | Cluster type (regional / zonal) | -| vertical\_pod\_autoscaling\_enabled | Whether veritical pod autoscaling is enabled | -| zones | List of zones in which the cluster resides | - ## Requirements @@ -269,6 +173,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/autogen/cluster.tf b/autogen/cluster.tf index 035eeb0cf7..87b11ca03c 100644 --- a/autogen/cluster.tf +++ b/autogen/cluster.tf @@ -333,22 +333,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -369,8 +361,8 @@ resource "google_container_node_pool" "pools" { } {% endif %} tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -433,6 +425,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/autogen/dns.tf b/autogen/dns.tf index d9d4a35395..20c3b25ee9 100644 --- a/autogen/dns.tf +++ b/autogen/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/autogen/sa.tf b/autogen/sa.tf index 62b31f457a..eaebeb2a22 100644 --- a/autogen/sa.tf +++ b/autogen/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/autogen/scripts/wait-for-cluster.sh b/autogen/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/autogen/scripts/wait-for-cluster.sh +++ b/autogen/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/autogen/variables.tf b/autogen/variables.tf index 8fd25a7fc8..ede2bac6c5 100644 --- a/autogen/variables.tf +++ b/autogen/variables.tf @@ -270,6 +270,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -305,6 +311,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} {% if private_cluster %} variable "deploy_using_private_endpoint" { diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 85139efe7d..379205db45 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -21,21 +21,209 @@ steps: - 'TF_VAR_org_id=$_ORG_ID' - 'TF_VAR_folder_id=$_FOLDER_ID' - 'TF_VAR_billing_account=$_BILLING_ACCOUNT' -- id: create +- id: create disable-client-cert-local + waitFor: + - prepare name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create'] -- id: converge + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create disable-client-cert-local'] +- id: converge disable-client-cert-local + waitFor: + - create disable-client-cert-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge'] -- id: verify + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge disable-client-cert-local'] +- id: verify disable-client-cert-local + waitFor: + - converge disable-client-cert-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify'] -- id: destroy + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify disable-client-cert-local'] +- id: destroy disable-client-cert-local + waitFor: + - verify disable-client-cert-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy'] + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy disable-client-cert-local'] +- id: create shared-vpc-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create shared-vpc-local'] +- id: converge shared-vpc-local + waitFor: + - create shared-vpc-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge shared-vpc-local'] +- id: verify shared-vpc-local + waitFor: + - converge shared-vpc-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify shared-vpc-local'] +- id: destroy shared-vpc-local + waitFor: + - verify shared-vpc-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy shared-vpc-local'] +- id: create simple-regional-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-local'] +- id: converge simple-regional-local + waitFor: + - create simple-regional-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-local'] +- id: verify simple-regional-local + waitFor: + - converge simple-regional-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-local'] +- id: destroy simple-regional-local + waitFor: + - verify simple-regional-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-local'] +- id: create simple-regional-private-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-regional-private-local'] +- id: converge simple-regional-private-local + waitFor: + - create simple-regional-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-regional-private-local'] +- id: verify simple-regional-private-local + waitFor: + - converge simple-regional-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-regional-private-local'] +- id: destroy simple-regional-private-local + waitFor: + - verify simple-regional-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-regional-private-local'] +- id: create simple-zonal-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-zonal-local'] +- id: converge simple-zonal-local + waitFor: + - create simple-zonal-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-local'] +- id: verify simple-zonal-local + waitFor: + - converge simple-zonal-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-local'] +- id: destroy simple-zonal-local + waitFor: + - verify simple-zonal-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-local'] +- id: create simple-zonal-private-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create simple-zonal-private-local'] +- id: converge simple-zonal-private-local + waitFor: + - create simple-zonal-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge simple-zonal-private-local'] +- id: verify simple-zonal-private-local + waitFor: + - converge simple-zonal-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify simple-zonal-private-local'] +- id: destroy simple-zonal-private-local + waitFor: + - verify simple-zonal-private-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy simple-zonal-private-local'] +- id: create stub-domains-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create stub-domains-local'] +- id: converge stub-domains-local + waitFor: + - create stub-domains-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge stub-domains-local'] +- id: verify stub-domains-local + waitFor: + - converge stub-domains-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify stub-domains-local'] +- id: destroy stub-domains-local + waitFor: + - verify stub-domains-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy stub-domains-local'] +- id: create upstream-nameservers-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create upstream-nameservers-local'] +- id: converge upstream-nameservers-local + waitFor: + - create upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge upstream-nameservers-local'] +- id: verify upstream-nameservers-local + waitFor: + - converge upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify upstream-nameservers-local'] +- id: destroy upstream-nameservers-local + waitFor: + - verify upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy upstream-nameservers-local'] +- id: create stub-domains-upstream-nameservers-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create stub-domains-upstream-nameservers-local'] +- id: converge stub-domains-upstream-nameservers-local + waitFor: + - create stub-domains-upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge stub-domains-upstream-nameservers-local'] +- id: verify stub-domains-upstream-nameservers-local + waitFor: + - converge stub-domains-upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify stub-domains-upstream-nameservers-local'] +- id: destroy stub-domains-upstream-nameservers-local + waitFor: + - verify stub-domains-upstream-nameservers-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy stub-domains-upstream-nameservers-local'] +- id: create workload-metadata-config-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create workload-metadata-config-local'] +- id: converge workload-metadata-config-local + waitFor: + - create workload-metadata-config-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge workload-metadata-config-local'] +- id: verify workload-metadata-config-local + waitFor: + - converge workload-metadata-config-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify workload-metadata-config-local'] +- id: destroy workload-metadata-config-local + waitFor: + - verify workload-metadata-config-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local'] tags: - 'ci' - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 3b7306297c..d5b2622e4f 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -24,4 +24,4 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.1.0' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.4.5' diff --git a/cluster.tf b/cluster.tf index ffdb27b0fc..d6f961d2b4 100644 --- a/cluster.tf +++ b/cluster.tf @@ -161,22 +161,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -184,8 +176,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -227,6 +219,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/dns.tf b/dns.tf index b240a23e65..8a581ff68e 100644 --- a/dns.tf +++ b/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/examples/deploy_service/README.md b/examples/deploy_service/README.md index 5dcb7ca7a7..e13981c450 100644 --- a/examples/deploy_service/README.md +++ b/examples/deploy_service/README.md @@ -37,7 +37,7 @@ It will: | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/deploy_service/outputs.tf b/examples/deploy_service/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/deploy_service/outputs.tf +++ b/examples/deploy_service/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/disable_client_cert/README.md b/examples/disable_client_cert/README.md index 14dd6545c0..2f531b9906 100644 --- a/examples/disable_client_cert/README.md +++ b/examples/disable_client_cert/README.md @@ -36,7 +36,7 @@ This example illustrates how to create a simple cluster and disable deprecated s | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/disable_client_cert/outputs.tf b/examples/disable_client_cert/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/disable_client_cert/outputs.tf +++ b/examples/disable_client_cert/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/node_pool/README.md b/examples/node_pool/README.md index 9215f091cb..237b3f0b6f 100644 --- a/examples/node_pool/README.md +++ b/examples/node_pool/README.md @@ -32,7 +32,7 @@ This example illustrates how to create a cluster with multiple custom node-pool | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/node_pool/outputs.tf b/examples/node_pool/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/node_pool/outputs.tf +++ b/examples/node_pool/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/shared_vpc/README.md b/examples/shared_vpc/README.md index 3b0f5a6157..964a346349 100644 --- a/examples/shared_vpc/README.md +++ b/examples/shared_vpc/README.md @@ -32,7 +32,7 @@ This example illustrates how to create a simple cluster where the host network i | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/shared_vpc/outputs.tf b/examples/shared_vpc/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/shared_vpc/outputs.tf +++ b/examples/shared_vpc/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_regional/README.md b/examples/simple_regional/README.md index fb209e47b5..4950b21e9a 100644 --- a/examples/simple_regional/README.md +++ b/examples/simple_regional/README.md @@ -14,6 +14,7 @@ This example illustrates how to create a simple cluster. | network | The VPC network to host the cluster in | string | n/a | yes | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | +| skip\_provisioners | Flag to skip local-exec provisioners | bool | `"false"` | no | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | ## Outputs @@ -31,7 +32,7 @@ This example illustrates how to create a simple cluster. | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 4662435fbd..353ae91906 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -35,6 +35,7 @@ module "gke" { ip_range_services = var.ip_range_services create_service_account = false service_account = var.compute_engine_service_account + skip_provisioners = var.skip_provisioners } data "google_client_config" "default" { diff --git a/examples/simple_regional/outputs.tf b/examples/simple_regional/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/simple_regional/outputs.tf +++ b/examples/simple_regional/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_regional/variables.tf b/examples/simple_regional/variables.tf index 6121eab9ea..e7405d9e21 100644 --- a/examples/simple_regional/variables.tf +++ b/examples/simple_regional/variables.tf @@ -47,3 +47,8 @@ variable "compute_engine_service_account" { description = "Service account to associate to the nodes in the cluster" } +variable "skip_provisioners" { + type = bool + description = "Flag to skip local-exec provisioners" + default = false +} diff --git a/examples/simple_regional_beta/outputs.tf b/examples/simple_regional_beta/outputs.tf index ad152e186c..0d770aa809 100644 --- a/examples/simple_regional_beta/outputs.tf +++ b/examples/simple_regional_beta/outputs.tf @@ -29,6 +29,6 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_regional_private/README.md b/examples/simple_regional_private/README.md index 8175482731..917c097951 100644 --- a/examples/simple_regional_private/README.md +++ b/examples/simple_regional_private/README.md @@ -31,7 +31,7 @@ This example illustrates how to create a simple private cluster. | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_regional_private/outputs.tf b/examples/simple_regional_private/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/simple_regional_private/outputs.tf +++ b/examples/simple_regional_private/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_regional_private_beta/outputs.tf b/examples/simple_regional_private_beta/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/simple_regional_private_beta/outputs.tf +++ b/examples/simple_regional_private_beta/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_zonal/README.md b/examples/simple_zonal/README.md index 691f95c719..c086ea3a4b 100644 --- a/examples/simple_zonal/README.md +++ b/examples/simple_zonal/README.md @@ -31,7 +31,7 @@ This example illustrates how to create a simple cluster. | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_zonal/outputs.tf b/examples/simple_zonal/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/simple_zonal/outputs.tf +++ b/examples/simple_zonal/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/simple_zonal_private/README.md b/examples/simple_zonal_private/README.md index e576800d72..83cb7c575b 100644 --- a/examples/simple_zonal_private/README.md +++ b/examples/simple_zonal_private/README.md @@ -32,7 +32,7 @@ This example illustrates how to create a simple private cluster. | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/simple_zonal_private/outputs.tf b/examples/simple_zonal_private/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/simple_zonal_private/outputs.tf +++ b/examples/simple_zonal_private/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/stub_domains/README.md b/examples/stub_domains/README.md index 126a1cd54c..bc4491b880 100644 --- a/examples/stub_domains/README.md +++ b/examples/stub_domains/README.md @@ -36,7 +36,7 @@ It will: | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/stub_domains/outputs.tf b/examples/stub_domains/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/stub_domains/outputs.tf +++ b/examples/stub_domains/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/stub_domains_private/README.md b/examples/stub_domains_private/README.md index ee4b89fa7f..205d5fdf76 100644 --- a/examples/stub_domains_private/README.md +++ b/examples/stub_domains_private/README.md @@ -38,7 +38,7 @@ It will: | network | | | project\_id | | | region | | -| service\_account | The service account to default running nodes as if not overridden in `node_pools`. | +| service\_account | The default service account used for running nodes. | | subnetwork | | | zones | List of zones in which the cluster resides | diff --git a/examples/stub_domains_private/outputs.tf b/examples/stub_domains_private/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/stub_domains_private/outputs.tf +++ b/examples/stub_domains_private/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/stub_domains_upstream_nameservers/outputs.tf b/examples/stub_domains_upstream_nameservers/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/stub_domains_upstream_nameservers/outputs.tf +++ b/examples/stub_domains_upstream_nameservers/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/upstream_nameservers/outputs.tf b/examples/upstream_nameservers/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/upstream_nameservers/outputs.tf +++ b/examples/upstream_nameservers/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/workload_metadata_config/main.tf b/examples/workload_metadata_config/main.tf index 11cae808d4..f9fb25da5b 100644 --- a/examples/workload_metadata_config/main.tf +++ b/examples/workload_metadata_config/main.tf @@ -40,8 +40,9 @@ module "gke" { subnetwork = var.subnetwork ip_range_pods = var.ip_range_pods ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account + create_service_account = true + grant_registry_access = true + registry_project_id = var.registry_project_id enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "172.16.0.0/28" diff --git a/examples/workload_metadata_config/outputs.tf b/examples/workload_metadata_config/outputs.tf index 0d972dcd88..01a13147c2 100644 --- a/examples/workload_metadata_config/outputs.tf +++ b/examples/workload_metadata_config/outputs.tf @@ -29,7 +29,7 @@ output "ca_certificate" { } output "service_account" { - description = "The service account to default running nodes as if not overridden in `node_pools`." + description = "The default service account used for running nodes." value = module.gke.service_account } diff --git a/examples/workload_metadata_config/variables.tf b/examples/workload_metadata_config/variables.tf index 040c78d2c4..eaa8c36e83 100644 --- a/examples/workload_metadata_config/variables.tf +++ b/examples/workload_metadata_config/variables.tf @@ -48,7 +48,6 @@ variable "ip_range_services" { description = "The secondary ip range to use for pods" } -variable "compute_engine_service_account" { - description = "Service account to associate to the nodes in the cluster" +variable "registry_project_id" { + description = "Project name for the GCR registry" } - diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 0fc0068e96..ba84c19cce 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -188,12 +188,14 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -258,6 +260,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 2348150d19..a74bd0ef6b 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -307,22 +307,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -341,8 +333,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -401,6 +393,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-private-cluster-update-variant/dns.tf b/modules/beta-private-cluster-update-variant/dns.tf index b240a23e65..8a581ff68e 100644 --- a/modules/beta-private-cluster-update-variant/dns.tf +++ b/modules/beta-private-cluster-update-variant/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index fc38644871..63bf31ac78 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -23,7 +23,7 @@ data "google_compute_zones" "available" { provider = google-beta project = var.project_id - region = var.region + region = local.region } resource "random_shuffle" "available_zones" { @@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" { locals { // location location = var.regional ? var.region : var.zones[0] + region = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) // kuberentes version diff --git a/modules/beta-private-cluster-update-variant/networks.tf b/modules/beta-private-cluster-update-variant/networks.tf index 14ea500e03..2456654130 100644 --- a/modules/beta-private-cluster-update-variant/networks.tf +++ b/modules/beta-private-cluster-update-variant/networks.tf @@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" { provider = google-beta name = var.subnetwork - region = var.region + region = local.region project = local.network_project_id } diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh +++ b/modules/beta-private-cluster-update-variant/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 9a869a830f..2585cf4ba8 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -40,7 +40,8 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null } variable "zones" { @@ -267,6 +268,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -302,6 +309,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} variable "deploy_using_private_endpoint" { type = bool diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 8782de87d2..d7415a8af6 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -190,10 +190,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -258,6 +260,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 56f40ed17a..ce8a709a41 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -235,22 +235,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -269,8 +261,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -328,6 +320,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-private-cluster/dns.tf b/modules/beta-private-cluster/dns.tf index b240a23e65..8a581ff68e 100644 --- a/modules/beta-private-cluster/dns.tf +++ b/modules/beta-private-cluster/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/modules/beta-private-cluster/scripts/wait-for-cluster.sh b/modules/beta-private-cluster/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/modules/beta-private-cluster/scripts/wait-for-cluster.sh +++ b/modules/beta-private-cluster/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index d783248ea2..2585cf4ba8 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -268,6 +268,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -303,6 +309,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} variable "deploy_using_private_endpoint" { type = bool diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index d611d2b787..419c7d017f 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -181,10 +181,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | string | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -249,6 +251,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index e37b2b3b31..4a001fda85 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -230,22 +230,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -264,8 +256,8 @@ resource "google_container_node_pool" "pools" { } } tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -323,6 +315,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/beta-public-cluster/dns.tf b/modules/beta-public-cluster/dns.tf index b240a23e65..8a581ff68e 100644 --- a/modules/beta-public-cluster/dns.tf +++ b/modules/beta-public-cluster/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/modules/beta-public-cluster/scripts/wait-for-cluster.sh b/modules/beta-public-cluster/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/modules/beta-public-cluster/scripts/wait-for-cluster.sh +++ b/modules/beta-public-cluster/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 8d20d440e2..aec51dcf34 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -268,6 +268,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -303,6 +309,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} variable "istio" { description = "(Beta) Enable Istio addon" diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index e817361124..fa9cdb8852 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -174,10 +174,12 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | node\_version | The Kubernetes version of the node pools. Defaults kubernetes_version (master) variable and can be overridden for individual node pools by setting the `version` key on them. Must be empyty or set the same as master at cluster creation. | string | `""` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | list(string) | `` | no | | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | -| region | The region to host the cluster in (required) | string | n/a | yes | +| region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -237,6 +239,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index e8db91a77a..c8ef96bb0b 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -238,22 +238,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -261,8 +253,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -305,6 +297,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/private-cluster-update-variant/dns.tf b/modules/private-cluster-update-variant/dns.tf index b240a23e65..8a581ff68e 100644 --- a/modules/private-cluster-update-variant/dns.tf +++ b/modules/private-cluster-update-variant/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index bfe746401c..2bd1c40d14 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -23,7 +23,7 @@ data "google_compute_zones" "available" { provider = google project = var.project_id - region = var.region + region = local.region } resource "random_shuffle" "available_zones" { @@ -34,6 +34,7 @@ resource "random_shuffle" "available_zones" { locals { // location location = var.regional ? var.region : var.zones[0] + region = var.region == null ? join("-", slice(split("-", var.zones[0]), 0, 2)) : var.region // for regional cluster - use var.zones if provided, use available otherwise, for zonal cluster use var.zones with first element extracted node_locations = var.regional ? coalescelist(compact(var.zones), sort(random_shuffle.available_zones.result)) : slice(var.zones, 1, length(var.zones)) // kuberentes version diff --git a/modules/private-cluster-update-variant/networks.tf b/modules/private-cluster-update-variant/networks.tf index a382073dc0..aae034eee5 100644 --- a/modules/private-cluster-update-variant/networks.tf +++ b/modules/private-cluster-update-variant/networks.tf @@ -27,6 +27,6 @@ data "google_compute_subnetwork" "gke_subnetwork" { provider = google name = var.subnetwork - region = var.region + region = local.region project = local.network_project_id } diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/modules/private-cluster-update-variant/sa.tf +++ b/modules/private-cluster-update-variant/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh +++ b/modules/private-cluster-update-variant/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 8008e08975..508a4f1b96 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -40,7 +40,8 @@ variable "regional" { variable "region" { type = string - description = "The region to host the cluster in (required)" + description = "The region to host the cluster in (optional if zonal cluster / required if regional)" + default = null } variable "zones" { @@ -257,6 +258,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -292,6 +299,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} variable "deploy_using_private_endpoint" { type = bool diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index dc0c16513c..5465544b82 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -176,8 +176,10 @@ In either case, upgrading to module version `v1.0.0` will trigger a recreation o | project\_id | The project ID to host the cluster in (required) | string | n/a | yes | | region | The region to host the cluster in (optional if zonal cluster / required if regional) | string | `"null"` | no | | regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | +| registry\_project\_id | Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project. | string | `""` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created. | string | `""` | no | +| skip\_provisioners | Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality. | bool | `"false"` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | map(list(string)) | `` | no | | subnetwork | The subnetwork to host the cluster in (required) | string | n/a | yes | | upstream\_nameservers | If specified, the values replace the nameservers taken by default from the node’s /etc/resolv.conf | list | `` | no | @@ -237,6 +239,9 @@ following project roles: - roles/iam.serviceAccountUser - roles/resourcemanager.projectIamAdmin (only required if `service_account` is set to `create`) +Additionally, if `service_account` is set to `create` and `grant_registry_access` is requested, the service account requires the following role on the `registry_project_id` project: +- roles/resourcemanager.projectIamAdmin + ### Enable APIs In order to operate with the Service Account you must activate the following APIs on the project where the Service Account was created: diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 412e8295ed..26c91c902b 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -166,22 +166,14 @@ resource "google_container_node_pool" "pools" { image_type = lookup(var.node_pools[count.index], "image_type", "COS") machine_type = lookup(var.node_pools[count.index], "machine_type", "n1-standard-2") labels = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_labels["all"], var.node_pools_labels[var.node_pools[count.index]["name"]], ) metadata = merge( - { - "cluster_name" = var.name - }, - { - "node_pool" = var.node_pools[count.index]["name"] - }, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, + lookup(lookup(var.node_pools_metadata, "default_values", {}), "node_pool", true) ? { "node_pool" = var.node_pools[count.index]["name"] } : {}, var.node_pools_metadata["all"], var.node_pools_metadata[var.node_pools[count.index]["name"]], { @@ -189,8 +181,8 @@ resource "google_container_node_pool" "pools" { }, ) tags = concat( - ["gke-${var.name}"], - ["gke-${var.name}-${var.node_pools[count.index]["name"]}"], + lookup(var.node_pools_tags, "default_values", [true, true])[0] ? ["gke-${var.name}"] : [], + lookup(var.node_pools_tags, "default_values", [true, true])[1] ? ["gke-${var.name}-${var.node_pools[count.index]["name"]}"] : [], var.node_pools_tags["all"], var.node_pools_tags[var.node_pools[count.index]["name"]], ) @@ -232,6 +224,7 @@ resource "google_container_node_pool" "pools" { } resource "null_resource" "wait_for_cluster" { + count = var.skip_provisioners ? 0 : 1 provisioner "local-exec" { command = "${path.module}/scripts/wait-for-cluster.sh ${var.project_id} ${var.name}" diff --git a/modules/private-cluster/dns.tf b/modules/private-cluster/dns.tf index b240a23e65..8a581ff68e 100644 --- a/modules/private-cluster/dns.tf +++ b/modules/private-cluster/dns.tf @@ -20,7 +20,7 @@ Delete default kube-dns configmap *****************************************/ resource "null_resource" "delete_default_kube_dns_configmap" { - count = local.custom_kube_dns_config || local.upstream_nameservers_config ? 1 : 0 + count = (local.custom_kube_dns_config || local.upstream_nameservers_config) && ! var.skip_provisioners ? 1 : 0 provisioner "local-exec" { command = "${path.module}/scripts/kubectl_wrapper.sh https://${local.cluster_endpoint} ${data.google_client_config.default.access_token} ${local.cluster_ca_certificate} ${path.module}/scripts/delete-default-resource.sh kube-system configmap kube-dns" diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/modules/private-cluster/scripts/wait-for-cluster.sh b/modules/private-cluster/scripts/wait-for-cluster.sh index 6ff3253d58..7bf9919b8f 100755 --- a/modules/private-cluster/scripts/wait-for-cluster.sh +++ b/modules/private-cluster/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2018 Google LLC +# Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,6 +15,10 @@ set -e +if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} +fi + PROJECT=$1 CLUSTER_NAME=$2 gcloud_command="gcloud container clusters list --project=$PROJECT --format=json" diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 28b744d868..508a4f1b96 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -258,6 +258,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -293,6 +299,11 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +} variable "deploy_using_private_endpoint" { type = bool diff --git a/sa.tf b/sa.tf index 9e063fcc22..c7f34e4fbb 100644 --- a/sa.tf +++ b/sa.tf @@ -64,7 +64,7 @@ resource "google_project_iam_member" "cluster_service_account-monitoring_viewer" resource "google_project_iam_member" "cluster_service_account-gcr" { count = var.create_service_account && var.grant_registry_access ? 1 : 0 - project = var.project_id + project = var.registry_project_id == "" ? var.project_id : var.registry_project_id role = "roles/storage.objectViewer" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } diff --git a/scripts/wait-for-cluster.sh b/scripts/wait-for-cluster.sh index 37f0176ec7..42c9841fec 100755 --- a/scripts/wait-for-cluster.sh +++ b/scripts/wait-for-cluster.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2019 Google LLC +# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ set -e if [ -n "${GOOGLE_APPLICATION_CREDENTIALS}" ]; then - CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} + export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE=${GOOGLE_APPLICATION_CREDENTIALS} fi PROJECT=$1 diff --git a/test/fixtures/shared/outputs.tf b/test/fixtures/shared/outputs.tf index 1c2eb5e9ba..71b4b250de 100644 --- a/test/fixtures/shared/outputs.tf +++ b/test/fixtures/shared/outputs.tf @@ -79,3 +79,6 @@ output "service_account" { value = module.example.service_account } +output "registry_project_id" { + value = var.registry_project_id +} diff --git a/test/fixtures/shared/variables.tf b/test/fixtures/shared/variables.tf index 76280e0065..5dff24dbd4 100644 --- a/test/fixtures/shared/variables.tf +++ b/test/fixtures/shared/variables.tf @@ -33,3 +33,7 @@ variable "compute_engine_service_account" { description = "The email address of the service account to associate with the GKE cluster" } +variable "registry_project_id" { + description = "Project to use for granting access to the GCR registry, if requested" +} + diff --git a/test/fixtures/simple_regional/example.tf b/test/fixtures/simple_regional/example.tf index a03fadb28b..7f8bb83637 100644 --- a/test/fixtures/simple_regional/example.tf +++ b/test/fixtures/simple_regional/example.tf @@ -25,4 +25,5 @@ module "example" { ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name compute_engine_service_account = var.compute_engine_service_account + skip_provisioners = true } diff --git a/test/fixtures/workload_metadata_config/example.tf b/test/fixtures/workload_metadata_config/example.tf index 4d4a98e119..3568cfa404 100644 --- a/test/fixtures/workload_metadata_config/example.tf +++ b/test/fixtures/workload_metadata_config/example.tf @@ -17,13 +17,13 @@ module "example" { source = "../../../examples/workload_metadata_config" - project_id = var.project_id - cluster_name_suffix = "-${random_string.suffix.result}" - region = var.region - zones = slice(var.zones, 0, 1) - network = google_compute_network.main.name - subnetwork = google_compute_subnetwork.main.name - ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name - ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name - compute_engine_service_account = var.compute_engine_service_account + project_id = var.project_id + cluster_name_suffix = "-${random_string.suffix.result}" + region = var.region + zones = slice(var.zones, 0, 1) + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + registry_project_id = var.registry_project_id } diff --git a/test/integration/workload_metadata_config/controls/gcloud.rb b/test/integration/workload_metadata_config/controls/gcloud.rb index e62606c78c..ad642ff7c9 100644 --- a/test/integration/workload_metadata_config/controls/gcloud.rb +++ b/test/integration/workload_metadata_config/controls/gcloud.rb @@ -13,8 +13,10 @@ # limitations under the License. project_id = attribute('project_id') +registry_project_id = attribute('registry_project_id') location = attribute('location') cluster_name = attribute('cluster_name') +service_account = attribute('service_account') control "gcloud" do title "Google Compute Engine GKE configuration" @@ -55,4 +57,20 @@ end end end + + describe command("gcloud projects get-iam-policy #{registry_project_id} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:iam) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + it "has expected registry roles" do + expect(iam['bindings']).to include("members" => ["serviceAccount:#{service_account}"], "role" => "roles/storage.objectViewer") + end + end end diff --git a/test/integration/workload_metadata_config/inspec.yml b/test/integration/workload_metadata_config/inspec.yml index f6f3811afa..4f2b7d40d6 100644 --- a/test/integration/workload_metadata_config/inspec.yml +++ b/test/integration/workload_metadata_config/inspec.yml @@ -9,3 +9,9 @@ attributes: - name: project_id required: true type: string + - name: service_account + required: true + type: string + - name: registry_project_id + required: false + type: string diff --git a/test/setup/make_source.sh b/test/setup/make_source.sh index b39944af41..ad3f57165a 100755 --- a/test/setup/make_source.sh +++ b/test/setup/make_source.sh @@ -19,6 +19,9 @@ echo "#!/usr/bin/env bash" > ../source.sh project_id=$(terraform output project_id) echo "export TF_VAR_project_id='$project_id'" >> ../source.sh +# We use the same project for registry project in the tests. +echo "export TF_VAR_registry_project_id='$project_id'" >> ../source.sh + sa_json=$(terraform output sa_key) # shellcheck disable=SC2086 echo "export SERVICE_ACCOUNT_JSON='$(echo $sa_json | base64 --decode)'" >> ../source.sh diff --git a/variables.tf b/variables.tf index 5edd4b3405..58cf1f4685 100644 --- a/variables.tf +++ b/variables.tf @@ -258,6 +258,12 @@ variable "grant_registry_access" { default = false } +variable "registry_project_id" { + type = string + description = "Project holding the Google Container Registry. If empty, we use the cluster project. If grant_registry_access is true, storage.objectViewer role is assigned on this project." + default = "" +} + variable "service_account" { type = string description = "The service account to run nodes as if not overridden in `node_pools`. The create_service_account variable default value (true) will cause a cluster-specific service account to be created." @@ -293,3 +299,8 @@ variable "cluster_resource_labels" { default = {} } +variable "skip_provisioners" { + type = bool + description = "Flag to skip all local-exec provisioners. It breaks `stub_domains` and `upstream_nameservers` variables functionality." + default = false +}