diff --git a/blueprints/apigee/hybrid-gke/README.md b/blueprints/apigee/hybrid-gke/README.md index cee4aec1a9..3058971621 100644 --- a/blueprints/apigee/hybrid-gke/README.md +++ b/blueprints/apigee/hybrid-gke/README.md @@ -25,20 +25,20 @@ The diagram below depicts the architecture. terraform apply ``` +Create an A record in your DNS registrar to point the environment group hostname to the public IP address returned after the terraform configuration was applied. You might need to wait some time until the certificate is provisioned. + ## Testing the blueprint 2. Deploy an api proxy ``` - ./deploy-apiproxy.sh + ./deploy-apiproxy.sh apis-test ``` -3. In the console check the IP address that has been allocated to the Apigee ingress gateway and send some traffic to the deployed API proxy. +3. Send a request ``` - curl -k -v -H "Host:HOSTNAME" \ - --resolve HOSTNAME:443:IP_ADDRESS \ - https://HOSTNAME/httpbin/headers + curl -v https://HOSTNAME/httpbin/headers ``` @@ -56,4 +56,10 @@ The diagram below depicts the architecture. | [region](variables.tf#L84) | Region. | string | | "europe-west1" | | [zone](variables.tf#L90) | Zone. | string | | "europe-west1-c" | +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [ip_address](outputs.tf#L17) | GLB IP address. | | + diff --git a/blueprints/apigee/hybrid-gke/ansible.tf b/blueprints/apigee/hybrid-gke/ansible.tf index e5a491a3c5..b7694ab1f2 100644 --- a/blueprints/apigee/hybrid-gke/ansible.tf +++ b/blueprints/apigee/hybrid-gke/ansible.tf @@ -18,12 +18,13 @@ resource "local_file" "vars_file" { content = yamlencode({ - cluster = module.cluster.name - region = var.region - project_id = module.project.project_id - envgroup = local.envgroup - env = local.environment - hostname = var.hostname + cluster = module.cluster.name + region = var.region + project_id = module.project.project_id + envgroups = local.envgroups + environments = local.environments + service_accounts = local.google_sas + ingress_ip_name = local.ingress_ip_name }) filename = "${path.module}/ansible/vars/vars.yaml" file_permission = "0666" diff --git a/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml new file mode 100644 index 0000000000..e74ca15969 --- /dev/null +++ b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/k8s_service_accounts.yaml @@ -0,0 +1,28 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Create and annotate k8s service account + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: "{{ k8s_service_account }}" + namespace: apigee + annotations: + iam.gke.io/gcp-service-account: "{{ google_service_account }}@{{ project_id }}.iam.gserviceaccount.com" + with_items: "{{ k8s_service_accounts }}" + loop_control: + loop_var: k8s_service_account \ No newline at end of file diff --git a/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml index 4b72039b8a..0907846fd4 100644 --- a/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml +++ b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/tasks/main.yaml @@ -1,11 +1,11 @@ # Copyright 2023 Google LLC -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,18 +19,27 @@ --project {{ project_id }} \ --internal-ip -- name: Install cert-manager - shell: > - kubectl apply \ - --validate=false \ - -f https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml +- name: Download cert-manager + uri: + url: https://github.com/jetstack/cert-manager/releases/download/v1.7.2/cert-manager.yaml + dest: ~/cert-manager.yaml -- name: Wait until pods are ready in cert-manager namespace - shell: > - kubectl wait --for=condition=ready pods \ - -l app.kubernetes.io/instance=cert-manager \ - -n cert-manager \ - --timeout=90s +- name: Apply metrics-server manifest to the cluster. + kubernetes.core.k8s: + state: present + src: ~/cert-manager.yaml + +- name: + kubernetes.core.k8s_info: + kind: Pod + wait: yes + label_selectors: + - "app.kubernetes.io/instance=cert-manager" + namespace: cert-manager + wait_timeout: 90 + wait_condition: + type: Ready + status: True - name: Fetch apigeectl version uri: @@ -48,7 +57,7 @@ unarchive: src: "~/apigeectl.tar.gz" dest: "~" - remote_src: yes + remote_src: yes - name: Move apigeectl folder shell: > @@ -66,25 +75,69 @@ file: src: ~/apigeectl/{{ item }} dest: "~/hybrid-files/{{ item }}" - state: link + state: link with_items: - tools - config - templates - - plugins + - plugins -- name: Create service accounts - shell: > - ~/hybrid-files/tools/create-service-account -i {{ project_id }} -e non-prod -d ~/hybrid-files/service-accounts +- name: Create apigee namespace + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: apigee + +- name: Create k8s service accounts + include_tasks: k8s_service_accounts.yaml + vars: + google_service_account: "{{ item.key }}" + k8s_service_accounts: "{{ item.value }}" + with_dict: "{{ service_accounts }}" -- name: Create certificates +- name: Set hostnames + set_fact: + hostnames: "{{ hostnames | default([]) + item.value }}" + with_dict: "{{ envgroups }}" + +- name: Create certificate and private key shell: > openssl req \ -nodes \ -new \ -x509 \ - -keyout ~/hybrid-files/certs/{{ envgroup }}.key \ - -out ~/hybrid-files/certs/{{ envgroup }}.cert -subj '/CN='{{ hostname }}'' -days 3650 + -keyout ~/hybrid-files/certs/server.key \ + -out ~/hybrid-files/certs/server.crt \ + -subj "/CN=apigee.com' \ + -addext "subjectAltName={{ hostnames | map('regex_replace', '^', 'DNS:') | join(',') }}"" + -days 3650 + +- name: Read certificate + slurp: + src: ~/hybrid-files/certs/server.crt + register: certificate_output + +- name: Read private ket + slurp: + src: ~/hybrid-files/certs/server.key + register: privatekey_output + +- name: Create secret + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Secret + metadata: + name: tls-hybrid-ingress + namespace: apigee + type: kubernetes.io/tls + data: + tls.crt: "{{ certificate_output.content }}" + tls.key: "{{ privatekey_output.content }}" - name: Create overrides.yaml template: @@ -96,48 +149,185 @@ curl -X POST -H "Authorization: Bearer $(gcloud auth print-access-token)" \ -H "Content-Type:application/json" \ "https://apigee.googleapis.com/v1/organizations/{{ project_id }}:setSyncAuthorization" \ - -d '{"identities":["'"serviceAccount:apigee-non-prod@{{ project_id }}.iam.gserviceaccount.com"'"]}' + -d '{"identities":["'"serviceAccount:apigee-synchronizer@{{ project_id }}.iam.gserviceaccount.com"'"]}' - name: Dry-run (init) shell: > - ~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client + ~/apigeectl/apigeectl init -f overrides/overrides.yaml --dry-run=client args: chdir: ~/hybrid-files - name: Install the Apigee deployment services Apigee Deployment Controller and Apigee Admission Webhook. shell: > - ~/apigeectl/apigeectl init -f overrides/overrides.yaml + ~/apigeectl/apigeectl init -f overrides/overrides.yaml args: - chdir: ~/hybrid-files + chdir: ~/hybrid-files -- name: Wait until pods are ready in apigee-system namespace - shell: > - kubectl wait --for=condition=ready pods \ - -l app=apigee-controller \ - -n apigee-system \ - --timeout=300s +- name: Wait for apigee-controller pod to be ready + kubernetes.core.k8s_info: + kind: Pod + wait: yes + label_selectors: + - "app=apigee-controller" + namespace: apigee-system + wait_timeout: 600 + wait_condition: + type: Ready + status: True -- name: Wait until pods are ready in apigee namespace - shell: > - kubectl wait --for=condition=ready pods \ - -l app=apigee-ingressgateway-manager \ - -n apigee \ - --timeout=300s +- name: Wait for apigee-selfsigned-issuer issuer to be ready + kubernetes.core.k8s_info: + kind: Issuer + wait: yes + name: apigee-selfsigned-issuer + namespace: apigee-system + wait_timeout: 600 + wait_condition: + type: Ready + status: True + +- name: Wait for apigee-serving-cert certificate to be ready + kubernetes.core.k8s_info: + kind: Certificate + wait: yes + name: apigee-serving-cert + namespace: apigee-system + wait_timeout: 600 + wait_condition: + type: Ready + status: True + +- name: Wait for apigee-resources-install job to be complete + kubernetes.core.k8s_info: + kind: Job + wait: yes + name: apigee-resources-install + namespace: apigee-system + wait_timeout: 360 + wait_condition: + type: Complete + status: True - name: Dry-run (apply) shell: > - ~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client + ~/apigeectl/apigeectl apply -f overrides/overrides.yaml --dry-run=client args: chdir: ~/hybrid-files - name: Install the Apigee runtime components shell: > - ~/apigeectl/apigeectl apply -f overrides/overrides.yaml + ~/apigeectl/apigeectl apply -f overrides/overrides.yaml args: - chdir: ~/hybrid-files + chdir: ~/hybrid-files -- name: Check status of the deployment - shell: > - while [ -n "$(kubectl get pods -n apigee | tail -n +2 | grep -v Running | grep -v Completed)" ]; do sleep 1; done - args: - chdir: ~/hybrid-files \ No newline at end of file +- name: Wait for apigee-runtime pod to be ready + kubernetes.core.k8s_info: + kind: Pod + wait: yes + label_selectors: + - "app=apigee-runtime" + namespace: apigee + wait_timeout: 360 + wait_condition: + type: Ready + status: True + +- name: + kubernetes.core.k8s: + state: present + definition: + apiVersion: apigee.cloud.google.com/v1alpha1 + kind: ApigeeRoute + metadata: + name: apigee-wildcard + namespace: apigee + spec: + hostnames: + - '*' + ports: + - number: 443 + protocol: HTTPS + tls: + credentialName: tls-hybrid-ingress + mode: SIMPLE + selector: + app: apigee-ingressgateway + enableNonSniClient: true + +- name: Create google-managed certificate + kubernetes.core.k8s: + state: present + definition: + apiVersion: networking.gke.io/v1 + kind: ManagedCertificate + metadata: + name: "apigee-cert-hybrid" + namespace: apigee + spec: + domains: "{{ hostnames }}" + +- name: Create backend config + kubernetes.core.k8s: + state: present + definition: + apiVersion: cloud.google.com/v1 + kind: BackendConfig + metadata: + name: apigee-ingress-backendconfig + namespace: apigee + spec: + healthCheck: + requestPath: /healthz/ready + port: 15021 + type: HTTP + logging: + enable: true + sampleRate: 0.5 + +- name: Create service + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Service + metadata: + name: apigee-ingressgateway-hybrid + namespace: apigee + annotations: + cloud.google.com/backend-config: '{"default": "apigee-ingress-backendconfig"}' + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/app-protocols: '{"https":"HTTPS", "status-port": "HTTP"}' + labels: + app: apigee-ingressgateway-hybrid + spec: + ports: + - name: status-port + port: 15021 + targetPort: 15021 + - name: https + port: 443 + targetPort: 8443 + selector: + app: apigee-ingressgateway + ingress_name: ingress + type: ClusterIP + +- name: Create ingress + kubernetes.core.k8s: + state: present + definition: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + annotations: + networking.gke.io/managed-certificates: "apigee-cert-hybrid" + kubernetes.io/ingress.global-static-ip-name: "{{ ingress_ip_name }}" + kubernetes.io/ingress.allow-http: "false" + name: xlb-apigee + namespace: apigee + spec: + defaultBackend: + service: + name: apigee-ingressgateway-hybrid + port: + number: 443 \ No newline at end of file diff --git a/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2 b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2 index 1c2c09ed8a..691cc6d5db 100644 --- a/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2 +++ b/blueprints/apigee/hybrid-gke/ansible/roles/apigee-hybrid/templates/overrides.yaml.j2 @@ -1,29 +1,26 @@ gcp: region: {{ region }} projectID: {{ project_id }} + workloadIdentityEnabled: true k8sCluster: name: {{ cluster }} - region: CLUSTER_LOCATION # Must be the closest Google Cloud region to your cluster. + region: {{ region }} # Must be the closest Google Cloud region to your cluster. org: {{ project_id }} -instanceID: "instance-1" +instanceID: "{{ cluster }}-{{ region }}" cassandra: hostNetwork: false - # Set to false for single region installations and multi-region installations - # with connectivity between pods in different clusters, for example GKE installations. - # Set to true for multi-region installations with no communication between - # pods in different clusters, for example GKE On-prem, GKE on AWS, Anthos on bare metal, - # AKS, EKS, and OpenShift installations. - # See Multi-region deployment: Prerequisites virtualhosts: - - name: {{ envgroup }} +{% for k in envgroups %} + - name: {{ k }} + sslSecret: tls-hybrid-ingress + additionalGateways: ["apigee-wildcard"] selector: app: apigee-ingressgateway - sslCertPath: ./certs/{{ envgroup }}.cert - sslKeyPath: ./certs/{{ envgroup }}.key +{% endfor %} ao: args: @@ -37,27 +34,9 @@ ingressGateways: replicaCountMax: 10 envs: - - name: {{ env }} - serviceAccountPaths: - synchronizer: ./service-accounts/{{ project_id }}-apigee-non-prod.json - udca: ./service-accounts/{{ project_id }}-apigee-non-prod.json - runtime: ./service-accounts/{{ project_id }}-apigee-non-prod.json - -mart: - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json - -connectAgent: - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json - -metrics: - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json - -udca: - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json - -watcher: - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json +{% for k in environments %} + - name: {{ k }} +{% endfor %} logger: - enabled: true - serviceAccountPath: ./service-accounts/{{ project_id }}-apigee-non-prod.json + enabled: false diff --git a/blueprints/apigee/hybrid-gke/apigee.tf b/blueprints/apigee/hybrid-gke/apigee.tf index e3dc6b2e6c..b92592aaba 100644 --- a/blueprints/apigee/hybrid-gke/apigee.tf +++ b/blueprints/apigee/hybrid-gke/apigee.tf @@ -15,8 +15,51 @@ */ locals { - envgroup = "test" - environment = "apis-test" + envgroups = { + test = [var.hostname] + } + environments = { + apis-test = { + envgroups = ["test"] + } + } + org_short_name = (length(module.project.project_id) < 16 ? + module.project.project_id : + substr(module.project.project_id, 0, 15)) + org_hash = format("%s-%s", local.org_short_name, substr(sha256(module.project.project_id), 0, 7)) + org_env_hashes = { + for k, v in local.environments : + k => format("%s-%s-%s", local.org_short_name, length(k) < 16 ? k : substr(k, 0, 15), substr(sha256("${module.project.project_id}:${k}"), 0, 7)) + } + google_sas = { + apigee-metrics = [ + "apigee-metrics-sa" + ] + apigee-cassandra = [ + "apigee-cassandra-schema-setup-${local.org_hash}-sa", + "apigee-cassandra-user-setup-${local.org_hash}-sa" + ] + apigee-mart = [ + "apigee-mart-${local.org_hash}-sa", + "apigee-connect-agent-${local.org_hash}-sa" + ] + apigee-watcher = [ + "apigee-watcher-${local.org_hash}-sa" + ] + apigee-udca = concat([ + "apigee-udca-${local.org_hash}-sa" + ], + [for k, v in local.org_env_hashes : + "apigee-udca-${local.org_env_hashes[k]}-sa" + ]) + apigee-synchronizer = [ + for k, v in local.org_env_hashes : + "apigee-synchronizer-${local.org_env_hashes[k]}-sa" + ] + apigee-runtime = [for k, v in local.org_env_hashes : + "apigee-runtime-${local.org_env_hashes[k]}-sa" + ] + } } module "apigee" { @@ -26,20 +69,24 @@ module "apigee" { analytics_region = var.region runtime_type = "HYBRID" } - envgroups = { - (local.envgroup) = [var.hostname] - } - environments = { - (local.environment) = { - envgroups = [local.envgroup] - } + envgroups = local.envgroups + environments = local.environments +} + +module "sas" { + for_each = local.google_sas + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = each.key + # authoritative roles granted *on* the service accounts to other identities + iam = { + "roles/iam.workloadIdentityUser" = [for v in each.value : "serviceAccount:${module.project.project_id}.svc.id.goog[apigee/${v}]"] } } resource "local_file" "deploy_apiproxy_file" { content = templatefile("${path.module}/templates/deploy-apiproxy.sh.tpl", { org = module.project.project_id - env = local.environment }) filename = "${path.module}/deploy-apiproxy.sh" file_permission = "0777" diff --git a/blueprints/apigee/hybrid-gke/diagram.png b/blueprints/apigee/hybrid-gke/diagram.png index 6d5c2d6bc9..57e07ca307 100644 Binary files a/blueprints/apigee/hybrid-gke/diagram.png and b/blueprints/apigee/hybrid-gke/diagram.png differ diff --git a/blueprints/apigee/hybrid-gke/glb.tf b/blueprints/apigee/hybrid-gke/glb.tf new file mode 100644 index 0000000000..80ff2269c2 --- /dev/null +++ b/blueprints/apigee/hybrid-gke/glb.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + ingress_ip_name = "apigee" +} + +module "addresses" { + source = "../../../modules/net-address" + project_id = module.project.project_id + global_addresses = [local.ingress_ip_name] +} diff --git a/blueprints/apigee/hybrid-gke/main.tf b/blueprints/apigee/hybrid-gke/main.tf index 5f1a676b2d..5be174ef55 100644 --- a/blueprints/apigee/hybrid-gke/main.tf +++ b/blueprints/apigee/hybrid-gke/main.tf @@ -40,5 +40,12 @@ module "project" { "roles/resourcemanager.projectIamAdmin" = [module.mgmt_server.service_account_iam_email] "roles/iam.serviceAccountAdmin" = [module.mgmt_server.service_account_iam_email] "roles/iam.serviceAccountKeyAdmin" = [module.mgmt_server.service_account_iam_email] + "roles/monitoring.metricWriter" = [module.sas["apigee-metrics"].iam_email] + "roles/storage.objectAdmin" = [module.sas["apigee-cassandra"].iam_email] + "roles/apigeeconnect.Agent" = [module.sas["apigee-mart"].iam_email] + "roles/apigee.runtimeAgent" = [module.sas["apigee-watcher"].iam_email] + "roles/apigee.analyticsAgent" = [module.sas["apigee-udca"].iam_email] + "roles/apigee.synchronizerManager" = [module.sas["apigee-synchronizer"].iam_email] + "roles/cloudtrace.agent" = [module.sas["apigee-runtime"].iam_email] } -} \ No newline at end of file +} diff --git a/blueprints/apigee/hybrid-gke/mgmt.tf b/blueprints/apigee/hybrid-gke/mgmt.tf index f51975f5f7..538940e7b9 100644 --- a/blueprints/apigee/hybrid-gke/mgmt.tf +++ b/blueprints/apigee/hybrid-gke/mgmt.tf @@ -34,4 +34,12 @@ module "mgmt_server" { type = var.mgmt_server_config.disk_type size = var.mgmt_server_config.disk_size } -} \ No newline at end of file + metadata = { + startup-script = <