From 4a283646fa80708308261f5b88c9c663ad4de8c9 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 29 Mar 2018 09:51:17 +0200 Subject: [PATCH 1/2] Tracking recent pod terminations --- pkg/apis/deployment/v1alpha/member_status.go | 42 +++++++++++++++ .../deployment/v1alpha/member_status_test.go | 53 +++++++++++++++++++ .../v1alpha/zz_generated.deepcopy.go | 7 +++ .../reconcile/action_rotate_member.go | 1 + .../reconcile/action_upgrade_member.go | 1 + pkg/deployment/resources/pod_inspector.go | 9 +++- 6 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 pkg/apis/deployment/v1alpha/member_status_test.go diff --git a/pkg/apis/deployment/v1alpha/member_status.go b/pkg/apis/deployment/v1alpha/member_status.go index a25c163c7..20831c25b 100644 --- a/pkg/apis/deployment/v1alpha/member_status.go +++ b/pkg/apis/deployment/v1alpha/member_status.go @@ -22,6 +22,12 @@ package v1alpha +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + // MemberStatus holds the current status of a single member (server) type MemberStatus struct { // ID holds the unique ID of the member. @@ -35,4 +41,40 @@ type MemberStatus struct { PodName string `json:"podName,omitempty"` // Conditions specific to this member Conditions ConditionList `json:"conditions,omitempty"` + // RecentTerminatons holds the times when this member was recently terminated. + // First entry is the oldest. (do not add omitempty, since we want to be able to switch from a list to an empty list) + RecentTerminations []metav1.Time `json:"recent-terminations"` +} + +// RemoveTerminationsBefore removes all recent terminations before the given timestamp. +// It returns the number of terminations that have been removed. +func (s *MemberStatus) RemoveTerminationsBefore(timestamp time.Time) int { + removed := 0 + for { + if len(s.RecentTerminations) == 0 { + // Nothing left + return removed + } + if s.RecentTerminations[0].Time.Before(timestamp) { + // Let's remove it + s.RecentTerminations = s.RecentTerminations[1:] + removed++ + } else { + // First (oldest) is not before given timestamp, we're done + return removed + } + } +} + +// RecentTerminationsSince returns the number of terminations since the given timestamp. +func (s MemberStatus) RecentTerminationsSince(timestamp time.Time) int { + count := 0 + for idx := len(s.RecentTerminations) - 1; idx >= 0; idx-- { + if s.RecentTerminations[idx].Time.Before(timestamp) { + // This termination is before the timestamp, so we're done + return count + } + count++ + } + return count } diff --git a/pkg/apis/deployment/v1alpha/member_status_test.go b/pkg/apis/deployment/v1alpha/member_status_test.go new file mode 100644 index 000000000..58594ed28 --- /dev/null +++ b/pkg/apis/deployment/v1alpha/member_status_test.go @@ -0,0 +1,53 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// TestMemberStatusRecentTerminations tests the functions related to MemberStatus.RecentTerminations. +func TestMemberStatusRecentTerminations(t *testing.T) { + relTime := func(delta time.Duration) metav1.Time { + return metav1.Time{Time: time.Now().Add(delta)} + } + + s := MemberStatus{} + assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(-time.Hour))) + assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now())) + + s.RecentTerminations = []metav1.Time{metav1.Now()} + assert.Equal(t, 1, s.RecentTerminationsSince(time.Now().Add(-time.Minute))) + assert.Equal(t, 0, s.RecentTerminationsSince(time.Now().Add(time.Minute))) + assert.Equal(t, 0, s.RemoveTerminationsBefore(time.Now().Add(-time.Hour))) + + s.RecentTerminations = []metav1.Time{relTime(-time.Hour), relTime(-time.Minute), relTime(time.Minute)} + assert.Equal(t, 3, s.RecentTerminationsSince(time.Now().Add(-time.Hour*2))) + assert.Equal(t, 2, s.RecentTerminationsSince(time.Now().Add(-time.Minute*2))) + assert.Equal(t, 2, s.RemoveTerminationsBefore(time.Now())) + assert.Len(t, s.RecentTerminations, 1) +} diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 6b32aaa0d..61ff324f7 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -371,6 +371,13 @@ func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.RecentTerminations != nil { + in, out := &in.RecentTerminations, &out.RecentTerminations + *out = make([]v1.Time, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/deployment/reconcile/action_rotate_member.go b/pkg/deployment/reconcile/action_rotate_member.go index 726abdde0..fb3e07bec 100644 --- a/pkg/deployment/reconcile/action_rotate_member.go +++ b/pkg/deployment/reconcile/action_rotate_member.go @@ -110,6 +110,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, error) { } // Pod is now gone, update the member status m.State = api.MemberStateNone + m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. if err := a.actionCtx.UpdateMember(m); err != nil { return false, maskAny(err) } diff --git a/pkg/deployment/reconcile/action_upgrade_member.go b/pkg/deployment/reconcile/action_upgrade_member.go index 8d5746a00..43611190c 100644 --- a/pkg/deployment/reconcile/action_upgrade_member.go +++ b/pkg/deployment/reconcile/action_upgrade_member.go @@ -120,6 +120,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) { } // Pod is now gone, update the member status m.State = api.MemberStateNone + m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. if err := a.actionCtx.UpdateMember(m); err != nil { return false, maskAny(err) } diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 5f5a5bb22..b419b3143 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -26,11 +26,12 @@ import ( "fmt" "time" - "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/metrics" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) var ( @@ -81,12 +82,18 @@ func (r *Resources) InspectPods() error { if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Succeeded", "") { log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Succeeded") updateMemberStatusNeeded = true + // Record termination time + now := metav1.Now() + memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now) } } else if k8sutil.IsPodFailed(&p) { // Pod has terminated with at least 1 container with a non-zero exit code. if memberStatus.Conditions.Update(api.ConditionTypeTerminated, true, "Pod Failed", "") { log.Debug().Str("pod-name", p.GetName()).Msg("Updating member condition Terminated to true: Pod Failed") updateMemberStatusNeeded = true + // Record termination time + now := metav1.Now() + memberStatus.RecentTerminations = append(memberStatus.RecentTerminations, now) } } if k8sutil.IsPodReady(&p) { From fbc0d1b84c078c48fe144d1779b26fda4ccb29fe Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Thu, 29 Mar 2018 10:11:50 +0200 Subject: [PATCH 2/2] Fix comment --- pkg/deployment/reconcile/action_upgrade_member.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/deployment/reconcile/action_upgrade_member.go b/pkg/deployment/reconcile/action_upgrade_member.go index 43611190c..f9fe9657b 100644 --- a/pkg/deployment/reconcile/action_upgrade_member.go +++ b/pkg/deployment/reconcile/action_upgrade_member.go @@ -120,7 +120,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) { } // Pod is now gone, update the member status m.State = api.MemberStateNone - m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. + m.RecentTerminations = nil // Since we're upgrading, we do not care about old terminations. if err := a.actionCtx.UpdateMember(m); err != nil { return false, maskAny(err) }