diff --git a/.kitchen.yml b/.kitchen.yml index c0d08c2418..6e9a7a24d8 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -52,6 +52,23 @@ suites: systems: - name: simple_regional backend: local + - name: "private_zonal_with_networking" + driver: + root_module_directory: test/fixtures/private_zonal_with_networking + verifier: + systems: + - name: private_zonal_with_networking + backend: local + controls: + - gcloud + - name: private_zonal_with_networking + backend: local + controls: + - subnet + - name: network + backend: gcp + controls: + - network - name: "simple_regional_with_networking" driver: root_module_directory: test/fixtures/simple_regional_with_networking diff --git a/examples/private_zonal_with_networking/main.tf b/examples/private_zonal_with_networking/main.tf new file mode 100644 index 0000000000..82db39a591 --- /dev/null +++ b/examples/private_zonal_with_networking/main.tf @@ -0,0 +1,87 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gcp-network" { + source = "terraform-google-modules/network/google" + version = "~> 1.4.0" + project_id = var.project_id + network_name = var.network + + subnets = [ + { + subnet_name = var.subnetwork + subnet_ip = "10.0.0.0/17" + subnet_region = var.region + subnet_private_access = "true" + }, + ] + + secondary_ranges = { + "${var.subnetwork}" = [ + { + range_name = var.ip_range_pods_name + ip_cidr_range = "192.168.0.0/18" + }, + { + range_name = var.ip_range_services_name + ip_cidr_range = "192.168.64.0/18" + }, + ] + } +} + +data "google_compute_subnetwork" "subnetwork" { + name = var.subnetwork + project = var.project_id + region = var.region + depends_on = [module.gcp-network] +} + +module "gke" { + source = "../../modules/beta-private-cluster/" + project_id = var.project_id + name = var.cluster_name + regional = false + region = var.region + zones = slice(var.zones, 0, 1) + + // This craziness gets a plain network name from the reference link which is the + // only way to force cluster creation to wait on network creation without a + // depends_on link. Tests use terraform 0.12.6, which does not have regex or regexall + network = reverse(split("/", data.google_compute_subnetwork.subnetwork.network))[0] + + subnetwork = data.google_compute_subnetwork.subnetwork.name + ip_range_pods = var.ip_range_pods_name + ip_range_services = var.ip_range_services_name + create_service_account = true + enable_private_endpoint = true + enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" + + master_authorized_networks_config = [ + { + cidr_blocks = [ + { + cidr_block = data.google_compute_subnetwork.subnetwork.ip_cidr_range + display_name = "VPC" + }, + ] + }, + ] +} + +data "google_client_config" "default" { +} diff --git a/examples/private_zonal_with_networking/outputs.tf b/examples/private_zonal_with_networking/outputs.tf new file mode 100644 index 0000000000..f306934801 --- /dev/null +++ b/examples/private_zonal_with_networking/outputs.tf @@ -0,0 +1,59 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + description = "The cluster endpoint" + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + description = "The bearer token for auth" + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + description = "The cluster ca certificate (base64 encoded)" + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.gcp-network.network_name +} + +output "subnet_name" { + description = "The name of the subnet being created" + value = module.gcp-network.subnets_names +} + +output "subnet_secondary_ranges" { + description = "The secondary ranges associated with the subnet" + value = module.gcp-network.subnets_secondary_ranges +} + + diff --git a/examples/private_zonal_with_networking/test_outputs.tf b/examples/private_zonal_with_networking/test_outputs.tf new file mode 100644 index 0000000000..e8eeacb9ef --- /dev/null +++ b/examples/private_zonal_with_networking/test_outputs.tf @@ -0,0 +1,58 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = var.project_id +} + +output "network" { + value = var.network +} + +output "subnetwork" { + value = var.subnetwork +} + +output "location" { + value = module.gke.location +} + +output "region" { + value = var.region +} + +output "ip_range_pods_name" { + description = "The secondary IP range used for pods" + value = var.ip_range_pods_name +} + +output "ip_range_services_name" { + description = "The secondary IP range used for services" + value = var.ip_range_services_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.gke.master_version +} diff --git a/examples/private_zonal_with_networking/variables.tf b/examples/private_zonal_with_networking/variables.tf new file mode 100644 index 0000000000..ecaf86a558 --- /dev/null +++ b/examples/private_zonal_with_networking/variables.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name" { + description = "The name for the GKE cluster" + default = "gke-on-vpc-cluster" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "zones" { + type = list(string) + description = "The zone to host the cluster in (required if is a zonal cluster)" +} + +variable "network" { + description = "The VPC network created to host the cluster in" + default = "gke-network" +} + +variable "subnetwork" { + description = "The subnetwork created to host the cluster in" + default = "gke-subnet" +} + +variable "ip_range_pods_name" { + description = "The secondary ip range to use for pods" + default = "ip-range-pods" +} + +variable "ip_range_services_name" { + description = "The secondary ip range to use for pods" + default = "ip-range-scv" +} + diff --git a/test/ci/private-zonal-with-networking.yml b/test/ci/private-zonal-with-networking.yml new file mode 100644 index 0000000000..f628957a28 --- /dev/null +++ b/test/ci/private-zonal-with-networking.yml @@ -0,0 +1,18 @@ +--- + +platform: linux + +inputs: +- name: pull-request + path: terraform-google-kubernetes-engine + +run: + path: make + args: ['test_integration'] + dir: terraform-google-kubernetes-engine + +params: + SUITE: "private-zonal-with-networking-local" + COMPUTE_ENGINE_SERVICE_ACCOUNT: "" + REGION: "us-east4" + ZONES: '["us-east4-a", "us-east4-b", "us-east4-c"]' diff --git a/test/fixtures/private_zonal_with_networking/example.tf b/test/fixtures/private_zonal_with_networking/example.tf new file mode 100644 index 0000000000..23612f1610 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/example.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2018 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/private_zonal_with_networking" + + project_id = var.project_id + region = var.region + zones = var.zones +} diff --git a/test/fixtures/private_zonal_with_networking/outputs.tf b/test/fixtures/private_zonal_with_networking/outputs.tf new file mode 100644 index 0000000000..08f9a8a2e8 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/outputs.tf @@ -0,0 +1,73 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +output "project_id" { + value = var.project_id +} + +output "location" { + value = module.example.location +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + value = module.example.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.example.service_account +} + +output "network_name" { + description = "The name of the VPC being created" + value = module.example.network +} + +output "subnet_name" { + description = "The name of the subnet being created" + value = module.example.subnetwork +} + +output "region" { + description = "The region the cluster is hosted in" + value = module.example.region +} + +output "ip_range_pods_name" { + description = "The secondary range name for pods" + value = module.example.ip_range_pods_name +} + +output "ip_range_services_name" { + description = "The secondary range name for services" + value = module.example.ip_range_services_name +} diff --git a/test/fixtures/private_zonal_with_networking/variables.tf b/test/fixtures/private_zonal_with_networking/variables.tf new file mode 100644 index 0000000000..d610f1e807 --- /dev/null +++ b/test/fixtures/private_zonal_with_networking/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "region" { + description = "The region to host the cluster in" + default = "us-east4" +} + +variable "zones" { + type = list(string) + description = "The GCP zones to create and test resources in, for applicable tests" + default = ["us-east4-a", "us-east4-b", "us-east4-c"] +} + diff --git a/test/integration/private_zonal_with_networking/controls/gcloud.rb b/test/integration/private_zonal_with_networking/controls/gcloud.rb new file mode 100644 index 0000000000..adaf6fd646 --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/gcloud.rb @@ -0,0 +1,101 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +location = attribute('location') +cluster_name = attribute('cluster_name') + +control "gcloud" do + title "Google Compute Engine GKE configuration" + describe command("gcloud --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + describe "cluster" do + it "is running" do + expect(data['status']).to eq 'RUNNING' + end + + it "is zonal" do + expect(data['location']).to match(/^(.*)[1-9]-[a-z]$/) + end + + it "is single zoned" do + expect(data['locations'].size).to eq 1 + end + + it "uses the private master endpoint" do + expect(data['privateClusterConfig']['enablePrivateEndpoint']).to eq true + end + + it "uses private nodes" do + expect(data['privateClusterConfig']['enablePrivateNodes']).to eq true + end + + it "has the expected addon settings" do + expect(data['addonsConfig']).to eq({ + "horizontalPodAutoscaling" => {}, + "httpLoadBalancing" => {}, + "kubernetesDashboard" => { + "disabled" => true, + }, + "networkPolicyConfig" => { + "disabled" => true, + }, + }) + end + end + + describe "default node pool" do + let(:default_node_pool) { data['nodePools'].select { |p| p['name'] == "default-pool" }.first } + + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "default-pool", + ) + ) + end + + it "is the expected machine type" do + expect(data['nodePools']).to include( + including( + "config" => including( + "machineType" => "n1-standard-1", + ), + ) + ) + end + + it "has the expected disk size" do + expect(data['nodePools']).to include( + including( + "config" => including( + "diskSizeGb" => 100, + ), + ) + ) + end + + end + end +end diff --git a/test/integration/private_zonal_with_networking/controls/network.rb b/test/integration/private_zonal_with_networking/controls/network.rb new file mode 100644 index 0000000000..a17ce74663 --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/network.rb @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +network_name = attribute('network_name') +subnet_name = attribute('subnet_name') +control "network" do + title "gcp network configuration" + describe google_compute_network( + project: project_id, + name: network_name + ) do + it { should exist } + its ('subnetworks.count') { should eq 1 } + its ('subnetworks.first') { should match subnet_name } + end + end diff --git a/test/integration/private_zonal_with_networking/controls/subnet.rb b/test/integration/private_zonal_with_networking/controls/subnet.rb new file mode 100644 index 0000000000..7b967b6c87 --- /dev/null +++ b/test/integration/private_zonal_with_networking/controls/subnet.rb @@ -0,0 +1,51 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +project_id = attribute('project_id') +network_name = attribute('network_name') +subnet_name = attribute('subnet_name') +region = attribute('region') +ip_range_pods_name = attribute('ip_range_pods_name') +ip_range_services_name = attribute('ip_range_services_name') + +control "subnet" do + title "gcp subnetwork configuration" + describe command("gcloud compute networks subnets describe #{subnet_name} --project=#{project_id} --region=#{region} --format=json") do + its(:exit_status) { should eq 0 } + its(:stderr) { should eq '' } + + let!(:data) do + if subject.exit_status == 0 + JSON.parse(subject.stdout) + else + {} + end + end + + it "#should have the correct secondaryIpRanges configuration for #{ip_range_pods_name}" do + expect(data["secondaryIpRanges"][0]).to include( + "rangeName" => ip_range_pods_name, + "ipCidrRange" => "192.168.0.0/18" + ) + end + + it "#should have the correct secondaryIpRanges configuration for #{ip_range_services_name}" do + expect(data["secondaryIpRanges"][1]).to include( + "rangeName" => ip_range_services_name, + "ipCidrRange" => "192.168.64.0/18" + ) + end + + end +end diff --git a/test/integration/private_zonal_with_networking/inspec.yml b/test/integration/private_zonal_with_networking/inspec.yml new file mode 100644 index 0000000000..87f447173f --- /dev/null +++ b/test/integration/private_zonal_with_networking/inspec.yml @@ -0,0 +1,36 @@ +name: private_zonal_with_networking +depends: + - name: inspec-gcp + git: https://github.com/inspec/inspec-gcp.git + tag: v0.10.0 +attributes: + - name: project_id + required: true + type: string + - name: location + required: true + type: string + - name: cluster_name + required: true + type: string + - name: kubernetes_endpoint + required: true + type: string + - name: client_token + required: true + type: string + - name: network_name + required: true + type: string + - name: subnet_name + required: true + type: string + - name: region + required: true + type: string + - name: ip_range_pods_name + required: true + type: string + - name: ip_range_services_name + required: true + type: string