diff --git a/.gitignore b/.gitignore index eda2ee0257..ec7ead32be 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,5 @@ examples/cloud-operations/binauthz/app/app.yaml env/ examples/cloud-operations/adfs/ansible/vars/vars.yaml examples/cloud-operations/adfs/ansible/gssh.sh +examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/vars.yaml +examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/gssh.sh diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/.gitignore b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/.gitignore new file mode 100644 index 0000000000..15c75f37d4 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/.gitignore @@ -0,0 +1,25 @@ +# terraform local cache directory +**/.terraform/* + +# terraform state files +*.tfstate +*.tfstate.backup + +# terraform lock file +*.terraform.lock.hcl + +# terraform variable files +*.tfvars + +# terraform backend configuration files +backend.tf +*.tfbackend + +# node modules +**/node_modules/* + +# python virtual environment +**/venv/* + +ansible/vars/vars.yaml +ansible/gssh.sh diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/README.md b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/README.md new file mode 100644 index 0000000000..73402b6cf9 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/README.md @@ -0,0 +1,65 @@ +# Multi-cluster mesh on GKE (fleet API) + +The following example shows how to create a multi-cluster mesh for two private clusters on GKE. Anthos Service Mesh with automatic control plane management is set up for clusters using the Fleet API. This can only be done if the clusters are in a single project and in the same VPC. In this particular case both clusters having being deployed to different subnets in a shared VPC. + +The diagram below depicts the architecture of the example. + +![Architecture](architecture.png) + +Terraform is used to provision the required infrastructure, create the IAM binding and register the clusters to the fleet. + +Ansible is used to execute commands in the management VM. From this VM there is access to the cluster's endpoint. More specifically the following is done using Ansible: + +1. Install required dependencies in the VM +2. Enable automatic control plane management in both clusters. +3. Verify the control plane has been provisioned for both clusters. +4. Configure ASM control plane endpoint discovery between the two clusters. +5. Create a sample namespace in both clusters. +6. Configure automatic sidecar injection in the created namespace. +7. Deploy a hello-world service in both clusters +8. Deploy a hello-world deployment (v1) in cluster a +9. Deploy a hello-world deployment (v2) in cluster b +10. Deploy a sleep service in both clusters. +11. Send requests from a sleep pod to the hello-world service from both clusters, to verify that we get responses from alternative versions. + +## Running the example + +Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=examples%2Fcloud-operations%2Fmulti-cluster-mesh-gke-fleet-api), then go through the following steps to create resources: + +* `terraform init` +* `terraform apply -var billing_account_id=my-billing-account-id -var parent=folders/my-folder-id -var host_project_id=my-host-project-id -var fleet_project_id=my-fleet-project-id -var mgmt_project_id=my-mgmt-project-id` + +Once terraform completes do the following: + +* Change to the ansible folder + + cd ansible + +* Run the ansible playbook + + ansible-playbook -v playbook.yaml + + +## Testing the example + +The last two commands executed with Ansible Send requests from a sleep pod to the hello-world service from both clusters. If you see in the output of those two commands responses from alternative versions, everything works as expected. + +Once done testing, you can clean up resources by running `terraform destroy`. + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | +| [fleet_project_id](variables.tf#L32) | Management Project ID. | string | ✓ | | +| [host_project_id](variables.tf#L27) | Project ID. | string | ✓ | | +| [mgmt_project_id](variables.tf#L37) | Management Project ID. | string | ✓ | | +| [parent](variables.tf#L22) | Parent. | string | ✓ | | +| [clusters_config](variables.tf#L54) | Clusters configuration. | map(object({…})) | | {…} | +| [istio_version](variables.tf#L98) | ASM version | string | | "1.14.1-asm.3" | +| [mgmt_server_config](variables.tf#L78) | Mgmt server configuration | object({…}) | | {…} | +| [mgmt_subnet_cidr_block](variables.tf#L42) | Management subnet CIDR block. | string | | "10.0.0.0/28" | +| [region](variables.tf#L48) | Region. | string | | "europe-west1" | + + diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/ansible.cfg b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/ansible.cfg new file mode 100644 index 0000000000..4558c2b6b1 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/ansible.cfg @@ -0,0 +1,9 @@ +[defaults] +inventory = inventory/hosts.ini +timeout = 900 + +[ssh_connection] +pipelining = True +ssh_executable = ./gssh.sh +transfer_method = piped + diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/inventory/hosts.ini b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..842da83f43 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/inventory/hosts.ini @@ -0,0 +1 @@ +mgmt \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/playbook.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/playbook.yaml new file mode 100644 index 0000000000..30114d22c8 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/playbook.yaml @@ -0,0 +1,27 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: mgmt + gather_facts: "no" + vars_files: + - vars/vars.yaml + environment: + USE_GKE_GCLOUD_AUTH_PLUGIN: True + roles: + - role: prerequisites + become: yes + become_method: sudo + - role: install + - role: test + diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/endpoint-discovery-config.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/endpoint-discovery-config.yaml new file mode 100644 index 0000000000..99abc8923d --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/endpoint-discovery-config.yaml @@ -0,0 +1,21 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Configure endpoint discovery + shell: > + kubectl apply \ + -f ~/{{ item }}.secret \ + --context "gke_{{ project_id }}_{{ region }}_{{ cluster }}" + with_items: "{{ clusters }}" + when: cluster != item \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/install.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/install.yaml new file mode 100644 index 0000000000..b81c49622a --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/install.yaml @@ -0,0 +1,58 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Get cluster credentials + shell: > + gcloud container clusters get-credentials {{ cluster }} \ + --region {{ region }} \ + --project {{ project_id }} \ + --internal-ip + +- name: Set context + set_fact: + context: "gke_{{ project_id }}_{{ region }}_{{ cluster }}" + +- name: Install ASM in cluster + shell: > + gcloud container fleet mesh update \ + --control-plane automatic \ + --memberships {{ cluster }} \ + --project {{ project_id }} + +- name: Wait until MCP is provisioned + shell: > + for i in $(seq 12); do + result=$(gcloud container fleet mesh describe --project {{ project_id }} --format json \ + | jq -r '.membershipStates | to_entries[] | select(.key | endswith("{{ cluster }}")) | .value.servicemesh.controlPlaneManagement.state') + if [ "$result" = "ACTIVE" ]; then + break + fi + echo "ASM control plane is not ready yet..." + sleep 60 + done + +- name: Get endpoint IP + shell: > + gcloud container clusters describe "{{ cluster }}" \ + --project "{{ project_id }}" \ + --region "{{ region }}" \ + --format "value(privateClusterConfig.publicEndpoint)" + register: endpoint + +- name: Create secret + shell: > + ~/istio-*/bin/istioctl x create-remote-secret \ + --context={{ context }} \ + --name={{ cluster }} \ + --server=https://{{ endpoint.stdout }} > ~/{{ cluster }}.secret \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/main.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/main.yaml new file mode 100644 index 0000000000..9e181bfbf7 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/install/tasks/main.yaml @@ -0,0 +1,39 @@ + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Download istio bundle + get_url: + url: https://storage.googleapis.com/gke-release/asm/istio-{{ istio_version }}-linux-amd64.tar.gz + dest: ~/istio.tar.gz + +- name: Unarchive istio bundle + unarchive: + src: ~/istio.tar.gz + dest: ~/ + remote_src: yes + +- name: Install + include_tasks: install.yaml + vars: + cluster: "{{ item }}" + with_items: "{{ clusters }}" + +- name: Configure endpoint discovery + include_tasks: endpoint-discovery-config.yaml + vars: + cluster: "{{ outer_item }}" + with_items: "{{ clusters }}" + loop_control: + loop_var: outer_item \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/prerequisites/tasks/main.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/prerequisites/tasks/main.yaml new file mode 100644 index 0000000000..8b889230ed --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/prerequisites/tasks/main.yaml @@ -0,0 +1,38 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Download the Google Cloud SDK package repository signing key + get_url: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + dest: /usr/share/keyrings/cloud.google.gpg + +- name: Add Google Cloud SDK package repository source + apt_repository: + filename: google-cloud-sdk.list + repo: "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" + state: present + update_cache: yes + +- name: Install dependencies + apt: + pkg: + - kubectl + - google-cloud-sdk-gke-gcloud-auth-plugin + - jq + state: present + +- name: Install gke-gcloud-auth-plugin + apt: + name: google-cloud-sdk-gke-gcloud-auth-plugin + state: present \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/main.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/main.yaml new file mode 100644 index 0000000000..46a06c16e0 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/main.yaml @@ -0,0 +1,36 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Deploy test app + include_tasks: test.yaml + vars: + cluster: "{{ item }}" + with_items: "{{ clusters }}" + loop_control: + index_var: index + +- name: Test + shell: > + for i in $(seq 400); do + kubectl exec \ + --context="gke_{{ project_id }}_{{ region }}_{{ item }}" \ + -n sample \ + -c sleep "$(kubectl get pod \ + --context="gke_{{ project_id }}_{{ region }}_{{ item }}" \ + -n sample \ + -l app=sleep \ + -o jsonpath='{.items[0].metadata.name}')" \ + -- curl -sS helloworld.sample:5000/hello + done + with_items: "{{ clusters }}" diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/test.yaml b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/test.yaml new file mode 100644 index 0000000000..2936907473 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/ansible/roles/test/tasks/test.yaml @@ -0,0 +1,63 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Set context + set_fact: + context: "gke_{{ project_id }}_{{ region }}_{{ cluster }}" + +- name: Create sample namespace + shell: + cmd: | + cat << EOF | kubectl apply --context {{ context }} -f - + apiVersion: v1 + kind: Namespace + metadata: + name: sample + EOF + +- name: Label the sample namespace for istio sidecar injection + shell: > + kubectl label namespace sample \ + istio-injection- istio.io/rev=asm-managed \ + --context {{ context }} \ + --overwrite + +- name: Create helloworld service + shell: > + kubectl apply + -f samples/helloworld/helloworld.yaml \ + -l service=helloworld \ + -n sample \ + --context {{ context }} + args: + chdir: ~/istio-{{ istio_version }} + +- name: Create helloworld deployment + shell: > + kubectl apply + -f samples/helloworld/helloworld.yaml \ + -l version=v{{ index + 1 }} \ + -n sample \ + --context {{ context }} + args: + chdir: ~/istio-{{ istio_version }} + +- name: Create sleep service and deployment + shell: > + kubectl apply \ + -f samples/sleep/sleep.yaml \ + -n sample \ + --context {{ context }} + args: + chdir: ~/istio-{{ istio_version }} \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/architecture.png b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/architecture.png new file mode 100644 index 0000000000..0838570f4a Binary files /dev/null and b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/architecture.png differ diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/main.tf b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/main.tf new file mode 100644 index 0000000000..034d7535cf --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/main.tf @@ -0,0 +1,263 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + np_service_account_iam_email = [for k, v in module.cluster_nodepools : v.service_account_iam_email] +} + +module "host_project" { + source = "../../../modules/project" + billing_account = var.billing_account_id + parent = var.parent + name = var.host_project_id + shared_vpc_host_config = { + enabled = true + service_projects = [] + } + services = [ + "container.googleapis.com" + ] +} + +module "mgmt_project" { + source = "../../../modules/project" + billing_account = var.billing_account_id + parent = var.parent + name = var.mgmt_project_id + shared_vpc_service_config = { + attach = true + host_project = module.host_project.project_id + service_identity_iam = null + } + services = [ + "cloudresourcemanager.googleapis.com", + "container.googleapis.com", + "serviceusage.googleapis.com" + ] +} + +module "fleet_project" { + source = "../../../modules/project" + billing_account = var.billing_account_id + parent = var.parent + name = var.fleet_project_id + shared_vpc_service_config = { + attach = true + host_project = module.host_project.project_id + service_identity_iam = { + "roles/compute.networkUser" = [ + "cloudservices", "container-engine" + ] + "roles/container.hostServiceAgentUser" = [ + "container-engine" + ] + } + } + services = [ + "anthos.googleapis.com", + "cloudresourcemanager.googleapis.com", + "container.googleapis.com", + "gkehub.googleapis.com", + "gkeconnect.googleapis.com", + "logging.googleapis.com", + "mesh.googleapis.com", + "monitoring.googleapis.com", + "stackdriver.googleapis.com" + ] + iam = { + "roles/container.admin" = [module.mgmt_server.service_account_iam_email] + "roles/gkehub.admin" = [module.mgmt_server.service_account_iam_email] + "roles/gkehub.serviceAgent" = ["serviceAccount:${module.fleet_project.service_accounts.robots.fleet}"] + "roles/monitoring.viewer" = local.np_service_account_iam_email + "roles/monitoring.metricWriter" = local.np_service_account_iam_email + "roles/logging.logWriter" = local.np_service_account_iam_email + "roles/stackdriver.resourceMetadata.writer" = local.np_service_account_iam_email + } + service_config = { + disable_on_destroy = false + disable_dependent_services = true + } +} + +module "svpc" { + source = "../../../modules/net-vpc" + project_id = module.host_project.project_id + name = "svpc" + mtu = 1500 + subnets = concat([for key, config in var.clusters_config : { + ip_cidr_range = config.subnet_cidr_block + name = "subnet-${key}" + region = var.region + secondary_ip_range = { + pods = config.pods_cidr_block + services = config.services_cidr_block + } + }], [{ + ip_cidr_range = var.mgmt_subnet_cidr_block + name = "subnet-mgmt" + region = var.mgmt_server_config.region + secondary_ip_range = null + }]) +} + +module "mgmt_server" { + source = "../../../modules/compute-vm" + project_id = module.mgmt_project.project_id + zone = var.mgmt_server_config.zone + name = "mgmt" + instance_type = var.mgmt_server_config.instance_type + network_interfaces = [{ + network = module.svpc.self_link + subnetwork = module.svpc.subnet_self_links["${var.mgmt_server_config.region}/subnet-mgmt"] + nat = false + addresses = null + }] + service_account_create = true + boot_disk = { + image = var.mgmt_server_config.image + type = var.mgmt_server_config.disk_type + size = var.mgmt_server_config.disk_size + } +} + +module "clusters" { + for_each = var.clusters_config + source = "../../../modules/gke-cluster" + project_id = module.fleet_project.project_id + name = each.key + location = var.region + network = module.svpc.self_link + subnetwork = module.svpc.subnet_self_links["${var.region}/subnet-${each.key}"] + secondary_range_pods = "pods" + secondary_range_services = "services" + private_cluster_config = { + enable_private_nodes = true + enable_private_endpoint = true + master_ipv4_cidr_block = each.value.master_cidr_block + master_global_access = true + } + master_authorized_ranges = merge({ + mgmt : var.mgmt_subnet_cidr_block + }, + { for key, config in var.clusters_config : + "pods-${key}" => config.pods_cidr_block if key != each.key + }) + enable_autopilot = false + release_channel = "REGULAR" + workload_identity = true + labels = { + mesh_id = "proj-${module.fleet_project.number}" + } +} + +module "cluster_nodepools" { + for_each = var.clusters_config + source = "../../../modules/gke-nodepool" + project_id = module.fleet_project.project_id + cluster_name = module.clusters[each.key].name + location = var.region + name = "nodepool-${each.key}" + node_service_account_create = true + initial_node_count = 1 + node_machine_type = "e2-standard-4" + node_tags = ["${each.key}-node"] +} + +module "firewall" { + source = "../../../modules/net-vpc-firewall" + project_id = module.host_project.project_id + network = module.svpc.name + custom_rules = merge({ allow-mesh = { + description = "Allow " + direction = "INGRESS" + action = "allow" + sources = [] + ranges = [for k, v in var.clusters_config : v.pods_cidr_block] + targets = [for k, v in var.clusters_config : "${k}-node"] + use_service_accounts = false + rules = [{ protocol = "tcp", ports = null }, + { protocol = "udp", ports = null }, + { protocol = "icmp", ports = null }, + { protocol = "esp", ports = null }, + { protocol = "ah", ports = null }, + { protocol = "sctp", ports = null }] + extra_attributes = { + priority = 900 + } + } }, + { for k, v in var.clusters_config : "allow-${k}-istio" => { + description = "Allow " + direction = "INGRESS" + action = "allow" + sources = [] + ranges = [v.master_cidr_block] + targets = ["${k}-node"] + use_service_accounts = false + rules = [{ protocol = "tcp", ports = [8080, 15014, 15017] }] + extra_attributes = { + priority = 1000 + } + } + } + ) +} + +module "nat" { + source = "../../../modules/net-cloudnat" + project_id = module.host_project.project_id + region = var.region + name = "nat" + router_create = true + router_network = module.svpc.name +} + +module "hub" { + source = "../../../modules/gke-hub" + project_id = module.fleet_project.project_id + clusters = { for k, v in module.clusters : k => v.id } + features = { + appdevexperience = false + configmanagement = false + identityservice = false + multiclusteringress = null + servicemesh = true + multiclusterservicediscovery = false + } + depends_on = [ + module.fleet_project + ] +} + +resource "local_file" "vars_file" { + content = templatefile("${path.module}/templates/vars.yaml.tpl", { + istio_version = var.istio_version + region = var.region + clusters = keys(var.clusters_config) + service_account_email = module.mgmt_server.service_account_email + project_id = module.fleet_project.project_id + }) + filename = "${path.module}/ansible/vars/vars.yaml" + file_permission = "0666" +} + +resource "local_file" "gssh_file" { + content = templatefile("${path.module}/templates/gssh.sh.tpl", { + project_id = var.mgmt_project_id + zone = var.mgmt_server_config.zone + }) + filename = "${path.module}/ansible/gssh.sh" + file_permission = "0777" +} diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/gssh.sh.tpl b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/gssh.sh.tpl new file mode 100644 index 0000000000..c61460ba2d --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/gssh.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +host="$${@: -2: 1}" +cmd="$${@: -1: 1}" + +gcloud_args=" +--tunnel-through-iap +--zone=${zone} +--project=${project_id} +--quiet +--no-user-output-enabled +-- +-C +" + +exec gcloud compute ssh "$host" $gcloud_args "$cmd" \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/vars.yaml.tpl b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/vars.yaml.tpl new file mode 100644 index 0000000000..f31b4fd752 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/templates/vars.yaml.tpl @@ -0,0 +1,8 @@ +istio_version: ${istio_version} +clusters: +%{ for cluster in clusters ~} + - ${cluster} +%{ endfor ~} +region: ${region} +service_account_email: ${service_account_email} +project_id: ${project_id} \ No newline at end of file diff --git a/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/variables.tf b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/variables.tf new file mode 100644 index 0000000000..a973a3ac62 --- /dev/null +++ b/examples/cloud-operations/multi-cluster-mesh-gke-fleet-api/variables.tf @@ -0,0 +1,102 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "billing_account_id" { + description = "Billing account id." + type = string +} + +variable "parent" { + description = "Parent." + type = string +} + +variable "host_project_id" { + description = "Project ID." + type = string +} + +variable "fleet_project_id" { + description = "Management Project ID." + type = string +} + +variable "mgmt_project_id" { + description = "Management Project ID." + type = string +} + +variable "mgmt_subnet_cidr_block" { + description = "Management subnet CIDR block." + type = string + default = "10.0.0.0/28" +} + +variable "region" { + description = "Region." + type = string + default = "europe-west1" +} + +variable "clusters_config" { + description = "Clusters configuration." + type = map(object({ + subnet_cidr_block = string + master_cidr_block = string + services_cidr_block = string + pods_cidr_block = string + })) + default = { + cluster-a = { + subnet_cidr_block = "10.0.1.0/24" + master_cidr_block = "10.16.0.0/28" + services_cidr_block = "192.168.1.0/24" + pods_cidr_block = "172.16.0.0/20" + } + cluster-b = { + subnet_cidr_block = "10.0.2.0/24" + master_cidr_block = "10.16.0.16/28" + services_cidr_block = "192.168.2.0/24" + pods_cidr_block = "172.16.16.0/20" + } + } +} + +variable "mgmt_server_config" { + description = "Mgmt server configuration" + type = object({ + disk_size = number + disk_type = string + image = string + instance_type = string + region = string + zone = string + }) + default = { + disk_size = 50 + disk_type = "pd-ssd" + image = "projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts" + instance_type = "n1-standard-2" + region = "europe-west1" + zone = "europe-west1-c" + } +} + +variable "istio_version" { + description = "ASM version" + type = string + default = "1.14.1-asm.3" +} diff --git a/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/__init__.py b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/__init__.py new file mode 100644 index 0000000000..6d6d1266c3 --- /dev/null +++ b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/main.tf b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/main.tf new file mode 100644 index 0000000000..77d4ba691e --- /dev/null +++ b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/main.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test" { + source = "../../../../../examples/cloud-operations/multi-cluster-mesh-gke-fleet-api" + billing_account_id = var.billing_account_id + parent = var.parent + host_project_id = var.host_project_id + fleet_project_id = var.fleet_project_id + mgmt_project_id = var.mgmt_project_id + region = var.region + clusters_config = var.clusters_config + mgmt_subnet_cidr_block = var.mgmt_subnet_cidr_block + istio_version = var.istio_version +} diff --git a/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/variables.tf b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/variables.tf new file mode 100644 index 0000000000..6c6b6c8fba --- /dev/null +++ b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/fixture/variables.tf @@ -0,0 +1,107 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "billing_account_id" { + description = "Billing account id." + type = string + default = "123-456-789" +} + +variable "parent" { + description = "Parent." + type = string + default = "folders/123456789" +} + +variable "host_project_id" { + description = "Host project ID." + type = string + default = "my-host-project" +} + +variable "fleet_project_id" { + description = "Fleet project ID." + type = string + default = "my-fleet-project" +} + +variable "mgmt_project_id" { + description = "Management Project ID." + type = string + default = "my-mgmt-project" +} + +variable "mgmt_subnet_cidr_block" { + description = "Management subnet CIDR block." + type = string + default = "10.0.0.0/24" +} + +variable "region" { + description = "Region." + type = string + default = "europe-west1" +} + +variable "clusters_config" { + description = "Clusters configuration." + type = map(object({ + subnet_cidr_block = string + master_cidr_block = string + services_cidr_block = string + pods_cidr_block = string + })) + default = { + cluster-a = { + subnet_cidr_block = "10.0.1.0/24" + master_cidr_block = "10.16.0.0/28" + services_cidr_block = "192.168.1.0/24" + pods_cidr_block = "172.16.0.0/20" + } + cluster-b = { + subnet_cidr_block = "10.0.2.0/24" + master_cidr_block = "10.16.0.16/28" + services_cidr_block = "192.168.2.0/24" + pods_cidr_block = "172.16.16.0/20" + } + } +} + +variable "mgmt_server_config" { + description = "Mgmt server configuration" + type = object({ + disk_size = number + disk_type = string + image = string + instance_type = string + region = string + zone = string + }) + default = { + disk_size = 50 + disk_type = "pd-ssd" + image = "projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts" + instance_type = "n1-standard-2" + region = "europe-west1" + zone = "europe-west1-c" + } +} + +variable "istio_version" { + description = "ASM version" + type = string + default = "1.14.1-asm.3" +} diff --git a/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/test_plan.py b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/test_plan.py new file mode 100644 index 0000000000..270a142d1d --- /dev/null +++ b/tests/examples/cloud_operations/multi_cluster_mesh_gke_fleet_api/test_plan.py @@ -0,0 +1,19 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def test_resources(e2e_plan_runner): + "Test that plan works and the numbers of resources is as expected." + modules, resources = e2e_plan_runner() + assert len(modules) == 12 + assert len(resources) == 53