diff --git a/.kitchen.yml b/.kitchen.yml index 6e9a7a24d8..10033a8a91 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -155,6 +155,19 @@ suites: systems: - name: workload_metadata_config backend: local + - name: "beta_cluster" + driver: + root_module_directory: test/fixtures/beta_cluster + verifier: + systems: + - name: gcloud + backend: local + controls: + - gcloud + - name: gcp + backend: gcp + controls: + - gcp - name: "deploy_service" driver: root_module_directory: test/fixtures/deploy_service diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index e78d0eb2ba..6982d474c8 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -244,6 +244,26 @@ steps: - verify workload-metadata-config-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-metadata-config-local'] +- id: create beta-cluster-local + waitFor: + - prepare + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do create beta-cluster-local'] +- id: converge beta-cluster-local + waitFor: + - create beta-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge beta-cluster-local'] +- id: verify beta-cluster-local + waitFor: + - converge beta-cluster-local + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify beta-cluster-local'] +#- id: destroy beta-cluster-local +# waitFor: +# - verify beta-cluster-local +# name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' +# args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy beta-cluster-local'] - id: create deploy-service-local waitFor: - prepare diff --git a/examples/simple_regional_beta/README.md b/examples/simple_regional_beta/README.md index 72bb221d9f..32bfc8fbfd 100644 --- a/examples/simple_regional_beta/README.md +++ b/examples/simple_regional_beta/README.md @@ -10,17 +10,22 @@ This example illustrates how to create a simple cluster with beta features. | cloudrun | Boolean to enable / disable CloudRun | string | `"true"` | no | | cluster\_name\_suffix | A suffix to append to the default cluster name | string | `""` | no | | compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | string | n/a | yes | +| database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key_name is the name of a CloudKMS key. | object | `` | no | +| enable\_binary\_authorization | Enable BinAuthZ Admission controller | string | `"false"` | no | | ip\_range\_pods | The secondary ip range to use for pods | string | n/a | yes | | ip\_range\_services | The secondary ip range to use for pods | string | n/a | yes | | istio | Boolean to enable / disable Istio | string | `"true"` | no | | network | The VPC network to host the cluster in | string | n/a | yes | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | string | `"SECURE"` | no | | node\_pools | List of maps containing node pools | list(map(string)) | `` | no | +| pod\_security\_policy\_config | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. | list | `` | no | | project\_id | The project ID to host the cluster in | string | n/a | yes | | region | The region to host the cluster in | string | n/a | yes | +| regional | Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!) | bool | `"true"` | no | | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | bool | `"false"` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` and `node_version` = `1.12.7-gke.17` or later to use it). | bool | `"false"` | no | | subnetwork | The subnetwork to host the cluster in | string | n/a | yes | +| zones | The zones to host the cluster in (optional if regional cluster / required if zonal) | list(string) | `` | no | ## Outputs diff --git a/examples/simple_regional_beta/main.tf b/examples/simple_regional_beta/main.tf index 0863cc51de..55acfaed79 100644 --- a/examples/simple_regional_beta/main.tf +++ b/examples/simple_regional_beta/main.tf @@ -24,23 +24,27 @@ provider "google-beta" { } module "gke" { - source = "../../modules/beta-public-cluster/" - project_id = var.project_id - name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" - regional = true - region = var.region - network = var.network - subnetwork = var.subnetwork - ip_range_pods = var.ip_range_pods - ip_range_services = var.ip_range_services - create_service_account = false - service_account = var.compute_engine_service_account - istio = var.istio - cloudrun = var.cloudrun - node_metadata = var.node_metadata - sandbox_enabled = var.sandbox_enabled - remove_default_node_pool = var.remove_default_node_pool - node_pools = var.node_pools + source = "../../modules/beta-public-cluster/" + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = var.regional + region = var.region + zones = var.zones + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = var.compute_engine_service_account == "create" + service_account = var.compute_engine_service_account + istio = var.istio + cloudrun = var.cloudrun + node_metadata = var.node_metadata + sandbox_enabled = var.sandbox_enabled + remove_default_node_pool = var.remove_default_node_pool + node_pools = var.node_pools + database_encryption = var.database_encryption + enable_binary_authorization = var.enable_binary_authorization + pod_security_policy_config = var.pod_security_policy_config } data "google_client_config" "default" { diff --git a/examples/simple_regional_beta/variables.tf b/examples/simple_regional_beta/variables.tf index ed16642774..58e1ae7433 100644 --- a/examples/simple_regional_beta/variables.tf +++ b/examples/simple_regional_beta/variables.tf @@ -85,3 +85,36 @@ variable "node_pools" { }, ] } + +variable "database_encryption" { + description = "Application-layer Secrets Encryption settings. The object format is {state = string, key_name = string}. Valid values of state are: \"ENCRYPTED\"; \"DECRYPTED\". key_name is the name of a CloudKMS key." + type = list(object({ state = string, key_name = string })) + default = [{ + state = "DECRYPTED" + key_name = "" + }] +} + +variable "enable_binary_authorization" { + description = "Enable BinAuthZ Admission controller" + default = false +} + +variable "pod_security_policy_config" { + description = "enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created." + default = [{ + "enabled" = false + }] +} + +variable "zones" { + type = list(string) + description = "The zones to host the cluster in (optional if regional cluster / required if zonal)" + default = [] +} + +variable "regional" { + type = bool + description = "Whether is a regional cluster (zonal cluster if set false. WARNING: changing this after cluster creation is destructive!)" + default = true +} diff --git a/test/ci/beta-cluster.yml b/test/ci/beta-cluster.yml new file mode 100644 index 0000000000..dd4ce29302 --- /dev/null +++ b/test/ci/beta-cluster.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "beta-cluster-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/beta_cluster/main.tf b/test/fixtures/beta_cluster/main.tf new file mode 100644 index 0000000000..4431b715f3 --- /dev/null +++ b/test/fixtures/beta_cluster/main.tf @@ -0,0 +1,78 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +provider "google" { + version = "~> 2.18.0" + project = var.project_id + region = var.region +} + +provider "google-beta" { + version = "~> 2.18.0" + project = var.project_id + region = var.region +} + +locals { + name = "beta-cluster-${random_string.suffix.result}" +} + +resource "google_kms_key_ring" "db" { + location = var.region + name = "${local.name}-db" +} + +resource "google_kms_crypto_key" "db" { + name = local.name + key_ring = google_kms_key_ring.db.self_link +} + +module "this" { + source = "../../../examples/simple_regional_beta" + + cluster_name_suffix = "-${random_string.suffix.result}" + project_id = var.project_id + regional = false + region = var.region + zones = slice(var.zones, 0, 1) + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = "create" + + // Beta features + istio = true + + database_encryption = [{ + state = "ENCRYPTED" + key_name = google_kms_crypto_key.db.self_link + }] + + cloudrun = true + + enable_binary_authorization = true + + pod_security_policy_config = [{ + enabled = true + }] + + node_metadata = "EXPOSE" +} + +data "google_client_config" "default" { +} diff --git a/test/fixtures/beta_cluster/network.tf b/test/fixtures/beta_cluster/network.tf new file mode 100644 index 0000000000..0a3f091958 --- /dev/null +++ b/test/fixtures/beta_cluster/network.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = var.region + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/beta_cluster/outputs.tf b/test/fixtures/beta_cluster/outputs.tf new file mode 100644 index 0000000000..da60cf87cf --- /dev/null +++ b/test/fixtures/beta_cluster/outputs.tf @@ -0,0 +1,84 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.this.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.this.cluster_name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.this.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.this.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.this.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.this.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.this.ca_certificate +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.this.service_account +} + +output "database_encryption_key_name" { + value = google_kms_crypto_key.db.self_link +} diff --git a/test/fixtures/beta_cluster/variables.tf b/test/fixtures/beta_cluster/variables.tf new file mode 120000 index 0000000000..c28fc18c01 --- /dev/null +++ b/test/fixtures/beta_cluster/variables.tf @@ -0,0 +1 @@ +../deploy_service/variables.tf \ No newline at end of file diff --git a/test/integration/beta_cluster/controls/gcloud.rb b/test/integration/beta_cluster/controls/gcloud.rb new file mode 100644 index 0000000000..455a81cb61 --- /dev/null +++ b/test/integration/beta_cluster/controls/gcloud.rb @@ -0,0 +1,204 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') +service_account = attribute('service_account') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses public nodes and master endpoint" do + expect(data['privateClusterConfig']).to eq nil + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + "istioConfig" => {}, + "cloudRunConfig" => {}, + }) + end + + it "has the expected binaryAuthorization config" do + expect(data['binaryAuthorization']).to eq({ + "enabled" => true, + }) + end + + it "has the expected nodeMetadata conseal config" do + expect(data['nodeConfig']['workloadMetadataConfig']).to eq({ + "nodeMetadata" => 'EXPOSE', + }) + end + + it "has the expected podSecurityPolicyConfig config" do + expect(data['podSecurityPolicyConfig']).to eq({ + "enabled" => true, + }) + end + + it "has the expected databaseEncryption config" do + expect(data['databaseEncryption']).to eq({ + "state" => 'ENCRYPTED', + "keyName" => attribute('database_encryption_key_name'), + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "has no initial node count" do + expect(default_node_pool['initialNodeCount']).to eq nil + end + + it "does not have autoscaling enabled" do + expect(default_node_pool['autoscaling']).to eq nil + end + end + + describe "node pool" do + let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" } } + + it "uses an automatically created service account" do + expect(node_pools).to include( + including( + "config" => including( + "serviceAccount" => service_account, + ), + ), + ) + end + + it "has autoscaling enabled" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "enabled" => true, + ), + ) + ) + end + + it "has the expected minimum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "minNodeCount" => 1, + ), + ) + ) + end + + it "has the expected maximum node count" do + expect(node_pools).to include( + including( + "autoscaling" => including( + "maxNodeCount" => 100, + ), + ) + ) + end + + it "is the expected machine type" do + expect(node_pools).to include( + including( + "config" => including( + "machineType" => "n1-standard-2", + ), + ) + ) + end + + it "has the expected disk size" do + expect(node_pools).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + it "has the expected labels" do + expect(node_pools).to include( + including( + "config" => including( + "labels" => including( + "cluster_name" => cluster_name, + "node_pool" => "default-node-pool", + ), + ), + ) + ) + end + + it "has the expected network tags" do + expect(node_pools).to include( + including( + "config" => including( + "tags" => match_array([ + "gke-#{cluster_name}", + "gke-#{cluster_name}-default-node-pool", + ]), + ), + ) + ) + end + + it "has autorepair enabled" do + expect(node_pools).to include( + including( + "management" => including( + "autoRepair" => true, + ), + ) + ) + end + end + end +end diff --git a/test/integration/beta_cluster/controls/gcp.rb b/test/integration/beta_cluster/controls/gcp.rb new file mode 100644 index 0000000000..6e9ade64ff --- /dev/null +++ b/test/integration/beta_cluster/controls/gcp.rb @@ -0,0 +1,31 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +control "gcp" do + title "Native InSpec Resources" + + service_account = attribute("service_account") + project_id = attribute("project_id") + + if service_account.start_with? "projects/" + service_account_name = service_account + else + service_account_name = "projects/#{project_id}/serviceAccounts/#{service_account}" + end + + describe google_service_account name: service_account_name do + its("display_name") { should eq "Terraform-managed service account for cluster #{attribute("cluster_name")}" } + its("project_id") { should eq project_id } + end +end diff --git a/test/integration/beta_cluster/inspec.yml b/test/integration/beta_cluster/inspec.yml new file mode 100644 index 0000000000..66062ea35d --- /dev/null +++ b/test/integration/beta_cluster/inspec.yml @@ -0,0 +1,33 @@ +name: beta_cluster +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: master_kubernetes_version + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: service_account + required: true + type: string + - name: service_account + required: true + type: string + - name: database_encryption_key_name + required: true + type: string diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 7ff4de74bc..a06a490c24 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -16,6 +16,7 @@ locals { int_required_roles = [ + "roles/cloudkms.admin", "roles/cloudkms.cryptoKeyEncrypterDecrypter", "roles/compute.networkAdmin", "roles/container.admin", @@ -58,3 +59,12 @@ resource "google_project_iam_member" "int_test" { resource "google_service_account_key" "int_test" { service_account_id = google_service_account.int_test.id } + +resource "google_project_iam_binding" "kubernetes_engine_kms_access" { + project = module.gke-project.project_id + role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" + + members = [ + "serviceAccount:service-${module.gke-project.project_number}@container-engine-robot.iam.gserviceaccount.com", + ] +} diff --git a/test/setup/main.tf b/test/setup/main.tf index 70e10c46a3..f974c7408e 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -24,8 +24,6 @@ module "gke-project" { folder_id = var.folder_id billing_account = var.billing_account - auto_create_network = true - activate_apis = [ "bigquery-json.googleapis.com", "cloudkms.googleapis.com",