feat(flagD): support zero downtime during upgrades (#731)

<!-- Please use this template for your pull request. -->
<!-- Please use the sections that you need and delete other sections -->

## This PR
<!-- add the description of the PR here -->

- implements graceful shutdown of flagD, which leads to zero-downtime ->
this means disabling the readiness probes and sending a shutdown event
to all connected SDKs
- create example manifests for deploying flagD as a standalone
Deployment
- create Makefile entry to deploy flagD to cluster
- create ZD test with README description how to run
- create Makefile entry to run ZD test

### Related Issues
<!-- add here the GitHub issue that this PR resolves if applicable -->

Fixes #728 

### Follow-up Tasks
- running ZD test as part of CI
#732

---------

Signed-off-by: odubajDT <[email protected]>

Makefile

@@ -1,8 +1,10 @@
-IMG ?= flagd:latest
 PHONY: .docker-build .build .run .mockgen
 PREFIX=/usr/local
 ALL_GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort)
 
+FLAGD_DEV_NAMESPACE ?= flagd-dev
+ZD_TEST_NAMESPACE ?= flagd-zd-test
+
 workspace-init: workspace-clean
 	go work init
 	$(foreach module, $(ALL_GO_MOD_DIRS), go work use $(module);)
@@ -67,6 +69,22 @@ mockgen: install-mockgen
 generate-docs:
 	cd flagd; go run ./cmd/doc/main.go
 
+.PHONY: deploy-dev-env
+export IMG?= ghcr.io/open-feature/flagd:latest
+deploy-dev-env: undeploy-dev-env
+	kubectl create ns "$(FLAGD_DEV_NAMESPACE)"
+	envsubst '$${IMG}' < config/deployments/flagd/deployment.yaml | kubectl apply -f - -n "$(FLAGD_DEV_NAMESPACE)"
+	kubectl apply -f config/deployments/flagd/service.yaml -n "$(FLAGD_DEV_NAMESPACE)"
+	kubectl wait --for=condition=available deployment/flagd -n "$(FLAGD_DEV_NAMESPACE)" --timeout=300s
+
+undeploy-dev-env:
+	kubectl delete ns "$(FLAGD_DEV_NAMESPACE)" --ignore-not-found=true
+
+run-zd-test:
+	kubectl delete ns "$(ZD_TEST_NAMESPACE)" --ignore-not-found=true
+	kubectl create ns "$(ZD_TEST_NAMESPACE)"
+	ZD_TEST_NAMESPACE="$(ZD_TEST_NAMESPACE)" FLAGD_DEV_NAMESPACE=$(FLAGD_DEV_NAMESPACE) IMG="$(IMG)" IMG_ZD="$(IMG_ZD)" ./test/zero-downtime/zd_test.sh
+
 # Markdown lint configuration
 #
 # - .markdownlintignore holds the configuration for files to be ignored

config/deployments/flagd/deployment.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: flagd
+  name: flagd
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app: flagd
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: flagd
+        app: flagd
+    spec:
+      containers:
+        - name: flagd
+          image: ${IMG}
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/flagd
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8014
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8014
+            initialDelaySeconds: 5
+            periodSeconds: 60
+          ports:
+            - containerPort: 8013
+          args:
+            - start
+            - --uri
+            - file:/etc/flagd/config.json
+            - --debug
+      volumes:
+        - name: config-volume
+          configMap:
+            name: open-feature-flags
+            items:
+              - key: flags
+                path: config.json
+---
+# ConfigMap for Flagd OpenFeatuer provider
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: open-feature-flags
+data:
+  flags: |
+    {
+      "flags": {
+        "myStringFlag": {
+          "state": "ENABLED",
+          "variants": {
+            "key1": "val1",
+            "key2": "val2"
+          },
+          "defaultVariant": "key1"
+        }
+      }
+    }

config/deployments/flagd/service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flagd-svc
+spec:
+  selector:
+    app.kubernetes.io/name: flagd
+  ports:
+    - port: 8013
+      targetPort: 8013

core/pkg/runtime/runtime.go

@@ -84,6 +84,13 @@ func (r *Runtime) Start() error {
 			return nil
 		})
 	}
+
+	defer func() {
+		r.Logger.Info("Shutting down server...")
+		r.Service.Shutdown()
+		r.Logger.Info("Server successfully shutdown.")
+	}()
+
 	g.Go(func() error {
 		// Readiness probe rely on the runtime
 		r.ServiceConfig.ReadinessProbe = r.isReady

core/pkg/service/flag-evaluation/connect_service.go

@@ -37,6 +37,8 @@ type ConnectService struct {
 
 	serverMtx        sync.RWMutex
 	metricsServerMtx sync.RWMutex
+
+	readinessEnabled bool
 }
 
 // NewConnectService creates a ConnectService with provided parameters
@@ -57,6 +59,7 @@ func NewConnectService(
 // Serve serves services with provided configuration options
 func (s *ConnectService) Serve(ctx context.Context, svcConf service.Configuration) error {
 	g, gCtx := errgroup.WithContext(ctx)
+	s.readinessEnabled = true
 
 	g.Go(func() error {
 		return s.startServer(svcConf)
@@ -152,6 +155,14 @@ func (s *ConnectService) AddMiddleware(mw middleware.IMiddleware) {
 	s.server.Handler = mw.Handler(s.server.Handler)
 }
 
+func (s *ConnectService) Shutdown() {
+	s.readinessEnabled = false
+	s.eventingConfiguration.emitToAll(service.Notification{
+		Type: service.Shutdown,
+		Data: map[string]interface{}{},
+	})
+}
+
 func (s *ConnectService) startServer(svcConf service.Configuration) error {
 	lis, err := s.setupServer(svcConf)
 	if err != nil {
@@ -189,7 +200,7 @@ func (s *ConnectService) startMetricsServer(svcConf service.Configuration) error
 		case "/healthz":
 			w.WriteHeader(http.StatusOK)
 		case "/readyz":
-			if svcConf.ReadinessProbe() {
+			if s.readinessEnabled && svcConf.ReadinessProbe() {
 				w.WriteHeader(http.StatusOK)
 			} else {
 				w.WriteHeader(http.StatusPreconditionFailed)

core/pkg/service/flag-evaluation/connect_service_test.go

@@ -204,3 +204,43 @@ func TestConnectServiceNotify(t *testing.T) {
 		t.Error("timeout while waiting for notifications")
 	}
 }
+
+func TestConnectServiceShutdown(t *testing.T) {
+	// given
+	ctrl := gomock.NewController(t)
+	eval := mock.NewMockIEvaluator(ctrl)
+
+	exp := metric.NewManualReader()
+	rs := resource.NewWithAttributes("testSchema")
+	metricRecorder := telemetry.NewOTelRecorder(exp, rs, "my-exporter")
+
+	service := NewConnectService(logger.NewLogger(nil, false), eval, metricRecorder)
+
+	sChan := make(chan iservice.Notification, 1)
+	eventing := service.eventingConfiguration
+	eventing.subs["key"] = sChan
+
+	// notification type
+	ofType := iservice.Shutdown
+
+	// emit notification in routine
+	go func() {
+		service.Notify(iservice.Notification{
+			Type: ofType,
+			Data: map[string]interface{}{},
+		})
+	}()
+
+	// wait for notification
+	timeout, cancelFunc := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancelFunc()
+
+	require.False(t, service.readinessEnabled)
+
+	select {
+	case n := <-sChan:
+		require.Equal(t, ofType, n.Type, "expected notification type: %s, but received %s", ofType, n.Type)
+	case <-timeout.Done():
+		t.Error("timeout while waiting for notifications")
+	}
+}

core/pkg/service/iservice.go

@@ -10,6 +10,7 @@ type NotificationType string
 
 const (
 	ConfigurationChange NotificationType = "configuration_change"
+	Shutdown            NotificationType = "provider_shutdown"
 	ProviderReady       NotificationType = "provider_ready"
 	KeepAlive           NotificationType = "keep_alive"
 )
@@ -40,6 +41,7 @@ which call the IEvaluator implementation.
 type IFlagEvaluationService interface {
 	Serve(ctx context.Context, svcConf Configuration) error
 	Notify(n Notification)
+	Shutdown()
 }
 
 /*

test/zero-downtime/README.md

@@ -0,0 +1,25 @@
+# FlagD Zero downtime test
+
+## How to run
+
+Clone this repository and run the following command to deploy a standalone flagD:
+
+```shell
+IMG=your-flagd-image make deploy-dev-env
+```
+
+This will create a flagd deployment `flagd-dev` namespace.
+
+To run the test, execute:
+
+```shell
+IMG=your-flagd-image IMG_ZD=your-flagd-image2 make run-zd-test
+```
+
+Please be aware, you need to build your two custom images with different tags for flagD first.
+
+To build your images using Docker execute:
+
+```shell
+docker build . -t image-name:tag -f flagd/build.Dockerfile
+```

test/zero-downtime/test-pod.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: test-zd
+spec:
+  containers:
+    - name: test-zd
+      image: curlimages/curl:8.1.2
+      # yamllint disable rule:line-length
+      command:
+        - 'sh'
+        - '-c'
+        - |
+          for i in $(seq 1 3000); do
+              curl -H 'Cache-Control: no-cache, no-store' -X POST flagd-svc.$FLAGD_DEV_NAMESPACE.svc.cluster.local:8013/schema.v1.Service/ResolveString?$RANDOM -d '{"flagKey":"myStringFlag","context":{}}' -H "Content-Type: application/json" > ~/out.txt
+              if ! grep -q "val1" ~/out.txt
+              then
+                cat ~/out.txt
+                echo "\n\nCannot fetch data from flagD, exiting...\n\n"
+                exit 1
+              fi
+              sleep 1
+          done
+          exit 0
+      # yamllint enable rule:line-length

test/zero-downtime/zd_test.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+set -eu
+
+# Store the flagD image to a helper variable
+IMG_ORIGINAL=$IMG
+
+# Create pod requesting the values from flagD
+envsubst < test/zero-downtime/test-pod.yaml | kubectl apply -f - -n $ZD_TEST_NAMESPACE
+
+for count in 1 2 3;
+do
+    # Update the flagD deployment with the second image
+    IMG=$IMG_ZD
+    envsubst < config/deployments/flagd/deployment.yaml | kubectl apply -f - -n $FLAGD_DEV_NAMESPACE
+    kubectl wait --for=condition=available deployment/flagd -n $FLAGD_DEV_NAMESPACE --timeout=30s
+
+    # Wait until the client pod executes curl requests agains flagD
+    sleep 20
+
+    # Update the flagDT deployment back to original image
+    IMG=$IMG_ORIGINAL
+    envsubst < config/deployments/flagd/deployment.yaml | kubectl apply -f - -n $FLAGD_DEV_NAMESPACE
+    kubectl wait --for=condition=available deployment/flagd -n $FLAGD_DEV_NAMESPACE --timeout=30s
+
+    # Wait until the client pod executes curl requests agains flagD
+    sleep 20
+done
+
+# Pod will fail only when it fails to get a proper response from curl (that means we do not have zero downtime)
+# If it is still running, the last curl request was successfull.
+kubectl wait --for=condition=ready pod/test-zd -n $ZD_TEST_NAMESPACE --timeout=30s
+
+# If curl request once not successful and another curl request was, pod might be in a ready state again.
+# Therefore we need to check that the restart count is equal to zero -> this means every request provided valid data.
+restart_count=$(kubectl get pods test-zd -o=jsonpath='{.status.containerStatuses[0].restartCount}' -n $ZD_TEST_NAMESPACE)
+if [ "$restart_count" -ne 0 ]; then
+    echo "Restart count of the test-zd pod is not equal to zero."
+    exit 1
+fi
+
+# Cleanup only when the test passed
+kubectl delete ns $ZD_TEST_NAMESPACE --ignore-not-found=true
+