Tracking recent pod terminations

pkg/apis/deployment/v1alpha/member_status.go

@@ -22,6 +22,12 @@
 
 package v1alpha
 
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
 // MemberStatus holds the current status of a single member (server)
 type MemberStatus struct {
 	// ID holds the unique ID of the member.
@@ -35,4 +41,40 @@ type MemberStatus struct {
 	PodName string `json:"podName,omitempty"`
 	// Conditions specific to this member
 	Conditions ConditionList `json:"conditions,omitempty"`
+	// RecentTerminatons holds the times when this member was recently terminated.
+	// First entry is the oldest. (do not add omitempty, since we want to be able to switch from a list to an empty list)
+	RecentTerminations []metav1.Time `json:"recent-terminations"`
+}
+
+// RemoveTerminationsBefore removes all recent terminations before the given timestamp.
+// It returns the number of terminations that have been removed.
+func (s *MemberStatus) RemoveTerminationsBefore(timestamp time.Time) int {
+	removed := 0
+	for {
+		if len(s.RecentTerminations) == 0 {
+			// Nothing left
+			return removed
+		}
+		if s.RecentTerminations[0].Time.Before(timestamp) {
+			// Let's remove it
+			s.RecentTerminations = s.RecentTerminations[1:]
+			removed++
+		} else {
+			// First (oldest) is not before given timestamp, we're done
+			return removed
+		}
+	}
+}
+
+// RecentTerminationsSince returns the number of terminations since the given timestamp.
+func (s MemberStatus) RecentTerminationsSince(timestamp time.Time) int {
+	count := 0
+	for idx := len(s.RecentTerminations) - 1; idx >= 0; idx-- {
+		if s.RecentTerminations[idx].Time.Before(timestamp) {
+			// This termination is before the timestamp, so we're done
+			return count
+		}
+		count++
+	}
+	return count
 }

pkg/apis/deployment/v1alpha/member_status_test.go

@@ -0,0 +1,53 @@
+//
+// DISCLAIMER
+//
+// Copyright 2018 ArangoDB GmbH, Cologne, Germany
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Copyright holder is ArangoDB GmbH, Cologne, Germany
+//
+// Author Ewout Prangsma
+//
+
+package v1alpha
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// TestMemberStatusRecentTerminations tests the functions related to MemberStatus.RecentTerminations.
+func TestMemberStatusRecentTerminations(t *testing.T) {
+	relTime := func(delta time.Duration) metav1.Time {
+		return metav1.Time{Time: time.Now().Add(delta)}
+	}
+
+	s := MemberStatus{}
+	assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(-time.Hour)))
+	assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now()))
+
+	s.RecentTerminations = []metav1.Time{metav1.Now()}
+	assert.Equal(t, 1, s.RecentTerminationsSince(time.Now().Add(-time.Minute)))
+	assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(time.Minute)))
+	assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now().Add(-time.Hour)))
+
+	s.RecentTerminations = []metav1.Time{relTime(-time.Hour), relTime(-time.Minute), relTime(time.Minute)}
+	assert.Equal(t, 3, s.RecentTerminationsSince(time.Now().Add(-time.Hour*2)))
+	assert.Equal(t, 2, s.RecentTerminationsSince(time.Now().Add(-time.Minute*2)))
+	assert.Equal(t, 2, s.RemoveTerminationsBefore(time.Now()))
+	assert.Len(t, s.RecentTerminations, 1)
+}

pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go

@@ -371,6 +371,13 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.RecentTerminations != nil {
+		in, out := &in.RecentTerminations, &out.RecentTerminations
+		*out = make([]v1.Time, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 

pkg/deployment/reconcile/action_rotate_member.go

@@ -110,6 +110,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, error) {
 	}
 	// Pod is now gone, update the member status
 	m.State = api.MemberStateNone
+	m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations.
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, maskAny(err)
 	}

pkg/deployment/reconcile/action_upgrade_member.go

@@ -120,6 +120,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) {
 	}
 	// Pod is now gone, update the member status
 	m.State = api.MemberStateNone
+	m.RecentTerminations = nil // Since we're upgrading, we do not care about old terminations.
 	if err := a.actionCtx.UpdateMember(m); err != nil {
 		return false, maskAny(err)
 	}

pkg/deployment/resources/pod_inspector.go

@@ -26,11 +26,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha"
 	"github.com/arangodb/kube-arangodb/pkg/metrics"
+	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
 )
 
 var (
@@ -81,12 +82,18 @@ func (r *Resources) InspectPods() error {
 			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Succeeded", "") {
 				log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Succeeded")
 				updateMemberStatusNeeded = true
+				// Record termination time
+				now := metav1.Now()
+				memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now)
 			}
 		} else if k8sutil.IsPodFailed(&p) {
 			// Pod has terminated with at least 1 container with a non-zero exit code.
 			if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Failed", "") {
 				log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Failed")
 				updateMemberStatusNeeded = true
+				// Record termination time
+				now := metav1.Now()
+				memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now)
 			}
 		}
 		if k8sutil.IsPodReady(&p) {