Improve Multi-Region Active-Active profile

google/include/kubernetes-gke.mk

@@ -22,17 +22,16 @@ kube-gke:
 	  --maintenance-window=4:00 \
 	  --release-channel=regular \
 	  --cluster-version=latest
-	gcloud container clusters list
-	kubectl apply -f $(root)/google/include/ssd-storageclass-gke.yaml
-	gcloud config set project $(project)
+	gcloud container clusters list --filter "name=$(clusterName)" --location $(region) --project $(project)
 	gcloud container clusters get-credentials $(clusterName) --region $(region)
+	kubectl apply -f $(root)/google/include/ssd-storageclass-gke.yaml
 
 .PHONY: node-pool # create an additional Kubernetes node pool
 node-pool:
 	gcloud beta container node-pools create "pool-c3-standard-8" \
 	  --project $(project) \
-	  --cluster $(clusterName) \
 	  --region $(region) \
+	  --cluster $(clusterName) \
 	  --machine-type "c3-standard-8" \
 	  --disk-type "pd-ssd" \
 	  --spot \
@@ -56,7 +55,8 @@ clean-kube-gke: use-kube
 #	-kubectl delete pvc --all
 	@echo "Please check the console if all PVCs have been deleted: https://console.cloud.google.com/compute/disks?authuser=0&project=$(project)&supportedpurview=project"
 	gcloud container clusters delete $(clusterName) --region $(region) --async --quiet
-	gcloud container clusters list
+	gcloud container clusters list --filter "name=$(clusterName)" --location $(region) --project $(project)
+	kubectl config delete-context gke_$(project)_$(region)_$(clusterName)
 
 .PHONY: use-kube
 use-kube:

google/include/log-format-stackdriver.yaml

@@ -0,0 +1,22 @@
+zeebe:
+  env:
+    # Enable JSON logging for Google Cloud Stackdriver
+    - name: ZEEBE_LOG_APPENDER
+      value: Stackdriver
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICENAME
+      value: zeebe
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICEVERSION
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+zeebe-gateway:
+  env:
+    # Enable JSON logging for Google Cloud Stackdriver
+    - name: ZEEBE_LOG_APPENDER
+      value: Stackdriver
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICENAME
+      value: zeebe
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICEVERSION
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace

google/multi-region/active-active/Makefile

@@ -0,0 +1,12 @@
+
+meld-gcp-setup:
+	meld region0/Makefile gcp-setup/Makefile
+
+meld-makefiles:
+	meld region0/Makefile region1/Makefile
+
+meld-regions:
+	meld region0 region1
+
+meld-failover:
+	meld camunda-values.yaml region0/camunda-values-failover.yaml

google/multi-region/active-active/camunda-values.yaml

@@ -0,0 +1,174 @@
+# Chart values for the Camunda Platform 8 Helm chart.
+# This file deliberately contains only the values that differ from the defaults.
+# For changes and documentation, use your favorite diff tool to compare it with:
+# https://github.com/camunda/camunda-platform-helm/blob/main/charts/camunda-platform/values.yaml
+
+global:
+  # Multiregion options for Zeebe
+  #
+  ## WARNING: In order to get your multi-region setup covered by Camunda enterprise support you MUST get your configuration and run books reviewed by Camunda before going to production.
+  # This is necessary for us to be able to help you in case of outages, due to the complexity of operating multi-region setups and the dependencies to the underlying Kubernetes prerequisites.
+  # If you operate this in the wrong way you risk corruption and complete loss of all data especially in the dual-region case.
+  # If you can, consider three regions. Please, contact your customer success manager as soon as you start planning a multi-region setup.
+  # Camunda reserves the right to limit support if no review was done prior to launch or the review showed significant risks.
+  multiregion:
+    # number of regions that this Camunda Platform instance is stretched across
+    regions: 2
+  identity:
+    auth:
+      # Disable the Identity authentication
+      # it will fall back to basic-auth: demo/demo as default user
+      enabled: false
+  elasticsearch:
+    disableExporter: true
+
+operate:
+  env:
+    - name: CAMUNDA_OPERATE_BACKUP_REPOSITORYNAME
+      value: "camunda_backup"
+tasklist:
+  env:
+    - name: CAMUNDA_TASKLIST_BACKUP_REPOSITORYNAME
+      value: "camunda_backup"
+
+identity:
+  enabled: false
+
+optimize:
+  enabled: false
+
+connectors:
+  enabled: true
+  inbound:
+    mode: credentials
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "512M"
+    limits:
+      cpu: "1000m"
+      memory: "2Gi"
+  env:
+    - name: CAMUNDA_OPERATE_CLIENT_USERNAME
+      value: demo
+    - name: CAMUNDA_OPERATE_CLIENT_PASSWORD
+      value: demo
+
+zeebe:
+  clusterSize: 8
+  partitionCount: 8
+  replicationFactor: 4
+  env:
+    - name: ZEEBE_BROKER_DATA_SNAPSHOTPERIOD
+      value: "5m"
+    - name: ZEEBE_BROKER_DATA_DISKUSAGECOMMANDWATERMARK
+      value: "0.85"
+    - name: ZEEBE_BROKER_DATA_DISKUSAGEREPLICATIONWATERMARK
+      value: "0.87"
+    - name: ZEEBE_BROKER_CLUSTER_INITIALCONTACTPOINTS
+      value: "camunda-zeebe-0.camunda-zeebe.us-east1.svc.cluster.local:26502,camunda-zeebe-1.camunda-zeebe.us-east1.svc.cluster.local:26502,camunda-zeebe-2.camunda-zeebe.us-east1.svc.cluster.local:26502,camunda-zeebe-3.camunda-zeebe.us-east1.svc.cluster.local:26502,camunda-zeebe-0.camunda-zeebe.europe-west1.svc.cluster.local:26502,camunda-zeebe-1.camunda-zeebe.europe-west1.svc.cluster.local:26502,camunda-zeebe-2.camunda-zeebe.europe-west1.svc.cluster.local:26502,camunda-zeebe-3.camunda-zeebe.europe-west1.svc.cluster.local:26502"
+    - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCHREGION0_CLASSNAME
+      value: "io.camunda.zeebe.exporter.ElasticsearchExporter"
+    - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCHREGION0_ARGS_URL
+      value: "http://elasticsearch-master-hl.us-east1.svc.cluster.local:9200"
+    - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCHREGION1_CLASSNAME
+      value: "io.camunda.zeebe.exporter.ElasticsearchExporter"
+    - name: ZEEBE_BROKER_EXPORTERS_ELASTICSEARCHREGION1_ARGS_URL
+      value: "http://elasticsearch-master-hl.europe-west1.svc.cluster.local:9200"
+    # Enable JSON logging for Google Cloud Stackdriver
+    - name: ZEEBE_LOG_APPENDER
+      value: Stackdriver
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICENAME
+      value: zeebe
+    - name: ZEEBE_LOG_STACKDRIVER_SERVICEVERSION
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+  pvcSize: 1Gi
+
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "512M"
+    limits:
+      cpu: "512m"
+      memory: "2Gi"
+
+zeebe-gateway:
+  replicas: 1
+
+  resources:
+    requests:
+      cpu: "100m"
+      memory: "512M"
+    limits:
+      cpu: "1000m"
+      memory: "1Gi"
+
+  logLevel: ERROR
+
+elasticsearch:
+  enabled: true
+  #  imageTag: 7.17.3
+  master:
+    replicaCount: 2
+    resources:
+      requests:
+        cpu: "100m"
+        memory: "512M"
+      limits:
+        cpu: "1000m"
+        memory: "2Gi"
+    persistence:
+      size: 15Gi
+
+  initContainers:
+    - name: install-gcs-plugin
+      image: elasticsearch:7.17.10
+      securityContext:
+        privileged: true
+      command:
+        - sh
+      args:
+        - -c
+        - |
+          ./bin/elasticsearch-plugin install --batch repository-gcs
+          ./bin/elasticsearch-keystore add-file -f gcs.client.default.credentials_file ./key/gcs_backup_key.json
+          cp -a ./config/elasticsearch.keystore /tmp/keystore
+      volumeMounts:
+        - name: plugins
+          mountPath: /usr/share/elasticsearch/plugins
+        - name: gcs-backup-key
+          mountPath: /usr/share/elasticsearch/key
+        - name: keystore
+          mountPath: /tmp/keystore
+  extraVolumes:
+    - name: plugins
+      emptyDir: {}
+    - name: keystore
+      emptyDir: {}
+    - name: gcs-backup-key
+      secret:
+        secretName: gcs-backup-key
+  extraVolumeMounts:
+    - name: plugins
+      mountPath: /usr/share/elasticsearch/plugins
+      readOnly: false
+    - mountPath: /usr/share/elasticsearch/key
+      name: gcs-backup-key
+    - name: keystore
+      mountPath: /usr/share/elasticsearch/config/elasticsearch.keystore
+      subPath: elasticsearch.keystore
+  # Allow no backup for single node setups ??? Not included in es values.yaml
+  # clusterHealthCheckParams: "wait_for_status=yellow&timeout=1s"
+
+
+
+
+  # # Request smaller persistent volumes.
+  # volumeClaimTemplate:
+  #   accessModes: [ "ReadWriteOnce" ]
+  #   storageClassName: "standard"
+  #   resources:
+  #     requests:
+  #       storage: 15Gi

google/multi-region/active-active/dns-lb.yaml

@@ -1,24 +1,22 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    # TODO: Check whether AWS/Azure can use internal load balancers. Google
-    # can't, unfortunately.
-    # service.beta.kubernetes.io/aws-load-balancer-internal: "true"
-    # service.beta.kubernetes.io/azure-load-balancer-internal: "true"
-    # TODO Falko try this:
-    # cloud.google.com/load-balancer-type: "Internal"
-  labels:
-    k8s-app: kube-dns
   name: kube-dns-lb
   namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+  annotations:
+    # see: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
+    service.beta.kubernetes.io/aws-load-balancer-internal: "true"
+    service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+    # FIXME: find firewall configuration for using: networking.gke.io/load-balancer-type: "Internal"
 spec:
+  type: LoadBalancer
+  sessionAffinity: None
+  selector:
+    k8s-app: kube-dns
   ports:
   - name: dns
     port: 53
     protocol: UDP
     targetPort: 53
-  selector:
-    k8s-app: kube-dns
-  sessionAffinity: None
-  type: LoadBalancer