Merged in master

Makefile

@@ -10,6 +10,7 @@ DOCKERCLI := $(shell which docker)
 
 GOBUILDDIR := $(SCRIPTDIR)/.gobuild
 SRCDIR := $(SCRIPTDIR)
+CACHEVOL := $(PROJECT)-gocache
 BINDIR := $(ROOTDIR)/bin
 VENDORDIR := $(ROOTDIR)/deps
 
@@ -125,6 +126,9 @@ $(GOBUILDDIR):
 	@rm -f $(REPODIR) && ln -sf ../../../.. $(REPODIR)
 	GOPATH=$(GOBUILDDIR) $(PULSAR) go flatten -V $(VENDORDIR)
 
+$(CACHEVOL):
+	@docker volume create $(CACHEVOL)
+
 .PHONY: update-vendor
 update-vendor:
 	@mkdir -p $(GOBUILDDIR)
@@ -176,11 +180,13 @@ update-generated: $(GOBUILDDIR)
 verify-generated:
 	@${MAKE} -B -s VERIFYARGS=--verify-only update-generated
 
-$(BIN): $(GOBUILDDIR) $(SOURCES)
+$(BIN): $(GOBUILDDIR) $(CACHEVOL) $(SOURCES)
 	@mkdir -p $(BINDIR)
 	docker run \
 		--rm \
 		-v $(SRCDIR):/usr/code \
+		-v $(CACHEVOL):/usr/gocache \
+		-e GOCACHE=/usr/gocache \
 		-e GOPATH=/usr/code/.gobuild \
 		-e GOOS=linux \
 		-e GOARCH=amd64 \
@@ -214,6 +220,8 @@ run-unit-tests: $(GOBUILDDIR) $(SOURCES)
 	docker run \
 		--rm \
 		-v $(SRCDIR):/usr/code \
+		-v $(CACHEVOL):/usr/gocache \
+		-e GOCACHE=/usr/gocache \
 		-e GOPATH=/usr/code/.gobuild \
 		-e GOOS=linux \
 		-e GOARCH=amd64 \
@@ -235,6 +243,8 @@ $(TESTBIN): $(GOBUILDDIR) $(SOURCES)
 	docker run \
 		--rm \
 		-v $(SRCDIR):/usr/code \
+		-v $(CACHEVOL):/usr/gocache \
+		-e GOCACHE=/usr/gocache \
 		-e GOPATH=/usr/code/.gobuild \
 		-e GOOS=linux \
 		-e GOARCH=amd64 \

docs/Manual/Deployment/Kubernetes/DeploymentResource.md

@@ -216,6 +216,42 @@ This setting specifies the pull policy for the docker image to use for all Arang
 For possible values, see `spec.imagePullPolicy`.
 When not specified, the `spec.imagePullPolicy` value is used.
 
+### `spec.sync.externalAccess.type: string`
+
+This setting specifies the type of `Service` that will be created to provide
+access to the ArangoSync syncMasters from outside the Kubernetes cluster.
+Possible values are:
+
+- `None` To limit access to applications running inside the Kubernetes cluster.
+- `LoadBalancer` To create a `Service` of type `LoadBalancer` for the ArangoSync SyncMasters.
+- `NodePort` To create a `Service` of type `NodePort` for the ArangoSync SyncMasters.
+- `Auto` (default) To create a `Service` of type `LoadBalancer` and fallback to a `Service` or type `NodePort` when the
+  `LoadBalancer` is not assigned an IP address.
+
+Note that when you specify a value of `None`, a `Service` will still be created, but of type `ClusterIP`.
+
+### `spec.sync.externalAccess.loadBalancerIP: string`
+
+This setting specifies the IP used for the LoadBalancer to expose the ArangoSync SyncMasters on.
+This setting is used when `spec.sync.externalAccess.type` is set to `LoadBalancer` or `Auto`.
+
+If you do not specify this setting, an IP will be chosen automatically by the load-balancer provisioner.
+
+### `spec.sync.externalAccess.nodePort: int`
+
+This setting specifies the port used to expose the ArangoSync SyncMasters on.
+This setting is used when `spec.sync.externalAccess.type` is set to `NodePort` or `Auto`.
+
+If you do not specify this setting, a random port will be chosen automatically.
+
+### `spec.sync.externalAccess.masterEndpoint: []string`
+
+This setting specifies the master endpoint(s) advertised by the ArangoSync SyncMasters.
+If not set, this setting defaults to:
+
+- If `spec.sync.externalAccess.loadBalancerIP` is set, it defaults to `https://<load-balancer-ip>:<8629>`.
+- Otherwise it defaults to `https://<sync-service-dns-name>:<8629>`.
+
 ### `spec.sync.auth.jwtSecretName: string`
 
 This setting specifies the name of a kubernetes `Secret` that contains

docs/design/lifecycle_hooks_and_finalizers.md

@@ -0,0 +1,37 @@
+# Lifecycle hooks & Finalizers
+
+The ArangoDB operator expects full control of the `Pods` and `PersistentVolumeClaims` it creates.
+Therefore it takes measures to prevent the removal of those resources
+until it is safe to do so.
+
+To achieve this, the server containers in the `Pods` have
+a `preStop` hook configured and finalizers are added to the `Pods`
+and `PersistentVolumeClaims`.
+
+The `preStop` hook executes a binary that waits until all finalizers of
+the current pod have been removed.
+Until this `preStop` hook terminates, Kubernetes will not send a `TERM` signal
+to the processes inside the container, which ensures that the server remains running
+until it is safe to stop them.
+
+The operator performs all actions needed when a delete of a `Pod` or
+`PersistentVolumeClaims` has been triggered.
+E.g. for a dbserver it cleans out the server if the `Pod` and `PersistentVolumeClaim` are being deleted.
+
+## Lifecycle init-container
+
+Because the binary that is called in the `preStop` hook is not part of a standard
+ArangoDB docker image, it has to be brought into the filesystem of a `Pod`.
+This is done by an initial container that copies the binary to an `emptyDir` volume that
+is shared between the init-container and the server container.
+
+## Finalizers
+
+The ArangoDB operators adds the following finalizers to `Pods`.
+
+- `dbserver.database.arangodb.com/drain`: Added to DBServers, removed only when the dbserver can be restarted or is completely drained
+- `agent.database.arangodb.com/agency-serving`: Added to Agents, removed only when enough agents are left to keep the agency serving
+
+The ArangoDB operators adds the following finalizers to `PersistentVolumeClaims`.
+
+- `pvc.database.arangodb.com/member-exists`: removed only when its member exists no longer exists or can be safely rebuild

docs/design/pod_evication_and_replacement.md

@@ -0,0 +1,124 @@
+# Pod Eviction & Replacement
+
+This chapter specifies the rules around evicting pods from nodes and
+restarting or replacing them.
+
+## Eviction
+
+Eviction is the process of removing a pod that is running on a node from that node.
+
+This is typically the result of a drain action (`kubectl drain`) or
+from a taint being added to a node (either automatically by Kubernetes or manually by an operator).
+
+## Replacement
+
+Replacement is the process of replacing a pod by another pod that takes over the responsibilities
+of the original pod.
+
+The replacement pod has a new ID and new (read empty) persistent data.
+
+Note that replacing a pod is different from restarting a pod. A pod is restarted when it has been reported
+to have termined.
+
+## NoExecute Tolerations
+
+NoExecute tolerations are used to control the behavior of Kubernetes (wrt. to a Pod) when the node
+that the pod is running on is no longer reachable or becomes not-ready.
+
+See the applicable [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) for more info.
+
+## Rules
+
+The rules for eviction & replacement are specified per type of pod.
+
+### Image ID Pods
+
+The Image ID pods are started to fetch the ArangoDB version of a specific
+ArangoDB image and fetch the docker sha256 of that image.
+They have no persistent state.
+
+- Image ID pods can always be evicted from any node
+- Image ID pods can always be restarted on a different node.
+  There is no need to replace an image ID pod, nor will it cause problems when
+  2 image ID pods run at the same time.
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set very low (5sec)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set very low (5sec)
+
+### Coordinator Pods
+
+Coordinator pods run an ArangoDB coordinator as part of an ArangoDB cluster.
+They have no persistent state, but do have a unique ID.
+
+- Coordinator pods can always be evicted from any node
+- Coordinator pods can always be replaced with another coordinator pod with a different ID on a different node
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set low (15sec)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set low (15sec)
+
+### DBServer Pods
+
+DBServer pods run an ArangoDB dbserver as part of an ArangoDB cluster.
+It has persistent state potentially tied to the node it runs on and it has a unique ID.
+
+- DBServer pods can be evicted from any node as soon as:
+  - It has been completely drained AND
+  - It is no longer the shard master for any shard
+- DBServer pods can be replaced with another dbserver pod with a different ID on a different node when:
+  - It is not the shard master for any shard OR
+  - For every shard it is the master for, there is an in-sync follower
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min)
+
+### Agent Pods
+
+Agent pods run an ArangoDB dbserver as part of an ArangoDB agency.
+It has persistent state potentially tight to the node it runs on and it has a unique ID.
+
+- Agent pods can be evicted from any node as soon as:
+  - It is no longer the agency leader AND
+  - There is at least an agency leader that is responding AND
+  - There is at least an agency follower that is responding
+- Agent pods can be replaced with another agent pod with the same ID but wiped persistent state on a different node when:
+  - The old pod is known to be deleted (e.g. explicit eviction)
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min)
+
+### Single Server Pods
+
+Single server pods run an ArangoDB server as part of an ArangoDB single server deployment.
+It has persistent state potentially tied to the node.
+
+- Single server pods cannot be evicted from any node.
+- Single server pods cannot be replaced with another pod.
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is not set to "wait it out forever"
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is not set "wait it out forever"
+
+### Single Pods in Active Failover Deployment
+
+Single pods run an ArangoDB single server as part of an ArangoDB active failover deployment.
+It has persistent state potentially tied to the node it runs on and it has a unique ID.
+
+- Single pods can be evicted from any node as soon as:
+  - It is a follower of an active-failover deployment (Q: can we trigger this failover to another server?)
+- Single pods can always be replaced with another single pod with a different ID on a different node.
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min)
+
+### SyncMaster Pods
+
+SyncMaster pods run an ArangoSync as master as part of an ArangoDB DC2DC cluster.
+They have no persistent state, but do have a unique address.
+
+- SyncMaster pods can always be evicted from any node
+- SyncMaster pods can always be replaced with another syncmaster pod on a different node
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set low (15sec)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set low (15sec)
+
+### SyncWorker Pods
+
+SyncWorker pods run an ArangoSync as worker as part of an ArangoDB DC2DC cluster.
+They have no persistent state, but do have in-memory state and a unique address.
+
+- SyncWorker pods can always be evicted from any node
+- SyncWorker pods can always be replaced with another syncworker pod on a different node
+- `node.kubernetes.io/unreachable:NoExecute` toleration time is set a bit higher to try to avoid resynchronization (1min)
+- `node.kubernetes.io/not-ready:NoExecute` toleration time is set a bit higher to try to avoid resynchronization (1min)

examples/cluster1-with-sync.yaml

@@ -0,0 +1,17 @@
+apiVersion: "database.arangodb.com/v1alpha"
+kind: "ArangoDeployment"
+metadata:
+  name: "cluster1-with-sync"
+spec:
+  mode: Cluster
+  image: ewoutp/arangodb:3.3.8
+  tls:
+    altNames: ["kube-01", "kube-02", "kube-03"]
+  sync:
+    enabled: true
+    auth:
+      clientCASecretName: client-auth-ca
+    externalAccess:
+      type: LoadBalancer
+      loadBalancerIP: 192.168.140.210
+

examples/cluster2-with-sync.yaml

@@ -0,0 +1,17 @@
+apiVersion: "database.arangodb.com/v1alpha"
+kind: "ArangoDeployment"
+metadata:
+  name: "cluster2-with-sync"
+spec:
+  mode: Cluster
+  image: ewoutp/arangodb:3.3.8
+  tls:
+    altNames: ["kube-01", "kube-02", "kube-03"]
+  sync:
+    enabled: true
+    auth:
+      clientCASecretName: client-auth-ca
+    externalAccess:
+      type: LoadBalancer
+      loadBalancerIP: 192.168.140.211
+