From 238f26ddc690f362e303741f63c5d69af3e43916 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 3 Apr 2018 10:58:05 +0200 Subject: [PATCH] Added LivenessProbe & Readiness probe --- main.go | 19 +++++--- .../templates/deployment/deployment.yaml | 17 +++++++ manifests/templates/storage/deployment.yaml | 18 +++++++ pkg/operator/operator.go | 13 +++-- pkg/operator/operator_deployment.go | 1 + pkg/operator/operator_local_storage.go | 1 + pkg/util/probe/health.go | 32 +++++++++++++ pkg/util/probe/ready.go | 48 +++++++++++++++++++ 8 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 pkg/util/probe/health.go create mode 100644 pkg/util/probe/ready.go diff --git a/main.go b/main.go index a066187a3..be3cee679 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/operator" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/probe" "github.com/arangodb/kube-arangodb/pkg/util/retry" ) @@ -80,6 +81,8 @@ var ( enableStorage bool // Run deployment operator createCRD bool } + deploymentProbe probe.Probe + storageProbe probe.Probe ) func init() { @@ -135,7 +138,9 @@ func cmdMainRun(cmd *cobra.Command, args []string) { cliLog.Fatal().Err(err).Msg("Failed to get hostname") } - //http.HandleFunc(probe.HTTPReadyzEndpoint, probe.ReadyzHandler) + http.HandleFunc("/health", probe.LivenessHandler) + http.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler) + http.HandleFunc("/ready/storage", storageProbe.ReadyHandler) http.Handle("/metrics", prometheus.Handler()) listenAddr := net.JoinHostPort(server.host, strconv.Itoa(server.port)) go http.ListenAndServe(listenAddr, nil) @@ -186,11 +191,13 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper CreateCRD: operatorOptions.createCRD, } deps := operator.Dependencies{ - LogService: logService, - KubeCli: kubecli, - KubeExtCli: kubeExtCli, - CRCli: crCli, - EventRecorder: eventRecorder, + LogService: logService, + KubeCli: kubecli, + KubeExtCli: kubeExtCli, + CRCli: crCli, + EventRecorder: eventRecorder, + DeploymentProbe: &deploymentProbe, + StorageProbe: &storageProbe, } return cfg, deps, nil diff --git a/manifests/templates/deployment/deployment.yaml b/manifests/templates/deployment/deployment.yaml index 09166b22b..b1f5fc385 100644 --- a/manifests/templates/deployment/deployment.yaml +++ b/manifests/templates/deployment/deployment.yaml @@ -6,6 +6,8 @@ metadata: namespace: {{ .Deployment.Operator.Namespace }} spec: replicas: 1 + strategy: + type: Recreate template: metadata: labels: @@ -26,3 +28,18 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + ports: + - name: metrics + containerPort: 8528 + livenessProbe: + httpGet: + path: /health + port: 8528 + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready/deployment + port: 8528 + initialDelaySeconds: 5 + periodSeconds: 10 diff --git a/manifests/templates/storage/deployment.yaml b/manifests/templates/storage/deployment.yaml index cd24b3a81..68605c1a1 100644 --- a/manifests/templates/storage/deployment.yaml +++ b/manifests/templates/storage/deployment.yaml @@ -14,6 +14,8 @@ metadata: namespace: {{ .Storage.Operator.Namespace }} spec: replicas: 1 + strategy: + type: Recreate template: metadata: labels: @@ -35,3 +37,19 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + ports: + - name: metrics + containerPort: 8528 + livenessProbe: + httpGet: + path: /health + port: 8528 + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready/storage + port: 8528 + initialDelaySeconds: 5 + periodSeconds: 10 + \ No newline at end of file diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 13d21b504..1a2b5fcfa 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -39,6 +39,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/logging" "github.com/arangodb/kube-arangodb/pkg/storage" + "github.com/arangodb/kube-arangodb/pkg/util/probe" ) const ( @@ -71,11 +72,13 @@ type Config struct { } type Dependencies struct { - LogService logging.Service - KubeCli kubernetes.Interface - KubeExtCli apiextensionsclient.Interface - CRCli versioned.Interface - EventRecorder record.EventRecorder + LogService logging.Service + KubeCli kubernetes.Interface + KubeExtCli apiextensionsclient.Interface + CRCli versioned.Interface + EventRecorder record.EventRecorder + DeploymentProbe *probe.Probe + StorageProbe *probe.Probe } // NewOperator instantiates a new operator from given config & dependencies. diff --git a/pkg/operator/operator_deployment.go b/pkg/operator/operator_deployment.go index f19b92560..be0b9e3c6 100644 --- a/pkg/operator/operator_deployment.go +++ b/pkg/operator/operator_deployment.go @@ -58,6 +58,7 @@ func (o *Operator) runDeployments(stop <-chan struct{}) { DeleteFunc: o.onDeleteArangoDeployment, }, cache.Indexers{}) + o.Dependencies.DeploymentProbe.SetReady() informer.Run(stop) } diff --git a/pkg/operator/operator_local_storage.go b/pkg/operator/operator_local_storage.go index d8e9138c1..db6e823e7 100644 --- a/pkg/operator/operator_local_storage.go +++ b/pkg/operator/operator_local_storage.go @@ -58,6 +58,7 @@ func (o *Operator) runLocalStorages(stop <-chan struct{}) { DeleteFunc: o.onDeleteArangoLocalStorage, }, cache.Indexers{}) + o.Dependencies.StorageProbe.SetReady() informer.Run(stop) } diff --git a/pkg/util/probe/health.go b/pkg/util/probe/health.go new file mode 100644 index 000000000..2ad9758de --- /dev/null +++ b/pkg/util/probe/health.go @@ -0,0 +1,32 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package probe + +import ( + "net/http" +) + +// LivenessHandler writes back the HTTP status code 200 to indicate a healthy operator. +func LivenessHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} diff --git a/pkg/util/probe/ready.go b/pkg/util/probe/ready.go new file mode 100644 index 000000000..636cc5a67 --- /dev/null +++ b/pkg/util/probe/ready.go @@ -0,0 +1,48 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package probe + +import ( + "net/http" + "sync/atomic" +) + +// Probe wraps a readiness probe handler. +type Probe struct { + ready int32 +} + +// SetReady marks the probe as ready. +func (p *Probe) SetReady() { + atomic.StoreInt32(&p.ready, 1) +} + +// ReadyHandler writes back the HTTP status code 200 if the operator is ready, and 500 otherwise. +func (p *Probe) ReadyHandler(w http.ResponseWriter, r *http.Request) { + isReady := atomic.LoadInt32(&p.ready) != 0 + if isReady { + w.WriteHeader(http.StatusOK) + } else { + w.WriteHeader(http.StatusInternalServerError) + } +}