From 5a2203d1f290d4bef8168526d4175357e1a05581 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 13:44:37 +0200 Subject: [PATCH 1/3] Detect pods not being scheduled --- examples/simple-cluster.yaml | 1 + pkg/deployment/resources/pod_inspector.go | 11 +++++++++++ pkg/util/k8sutil/pods.go | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/examples/simple-cluster.yaml b/examples/simple-cluster.yaml index 52fb63621..8891a71e8 100644 --- a/examples/simple-cluster.yaml +++ b/examples/simple-cluster.yaml @@ -5,6 +5,7 @@ metadata: spec: mode: cluster image: arangodb/arangodb:3.3.4 + environment: production tls: altNames: ["kube-01", "kube-02", "kube-03"] coordinators: diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index c6ecec8d8..3cf0963dd 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -23,6 +23,8 @@ package resources import ( + "time" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "k8s.io/api/core/v1" @@ -34,6 +36,10 @@ var ( inspectedPodCounter = metrics.MustRegisterCounter("deployment", "inspected_pods", "Number of pod inspections") ) +const ( + podScheduleTimeout = time.Minute // How long we allow the schedule to take scheduling a pod. +) + // InspectPods lists all pods that belong to the given deployment and updates // the member status of the deployment accordingly. func (r *Resources) InspectPods() error { @@ -49,6 +55,7 @@ func (r *Resources) InspectPods() error { // Update member status from all pods found status := r.context.GetStatus() apiObject := r.context.GetAPIObject() + podWithScheduleTimeout := 0 for _, p := range pods { if k8sutil.IsArangoDBImageIDAndVersionPod(p) { // Image ID pods are not relevant to inspect here @@ -93,6 +100,10 @@ func (r *Resources) InspectPods() error { updateMemberStatusNeeded = true } } + if k8sutil.IsPodNotScheduledFor(&p, podScheduleTimeout) { + // Pod cannot be scheduled for to long + podWithScheduleTimeout++ + } if updateMemberStatusNeeded { if err := status.Members.UpdateMemberStatus(memberStatus, group); err != nil { return maskAny(err) diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 8945ef13d..63a701d3f 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -25,6 +25,7 @@ package k8sutil import ( "fmt" "path/filepath" + "time" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -87,6 +88,15 @@ func IsPodFailed(pod *v1.Pod) bool { return pod.Status.Phase == v1.PodFailed } +// IsPodNotScheduledFor returns true if the pod has not been scheduled +// for longer than the given duration. +func IsPodNotScheduledFor(pod *v1.Pod, timeout time.Duration) bool { + condition := getPodCondition(&pod.Status, v1.PodScheduled) + return condition != nil && + condition.Status == v1.ConditionTrue && + condition.LastTransitionTime.Time.Add(timeout).After(time.Now()) +} + // IsArangoDBImageIDAndVersionPod returns true if the given pod is used for fetching image ID and ArangoDB version of an image func IsArangoDBImageIDAndVersionPod(p v1.Pod) bool { role, found := p.GetLabels()[LabelKeyRole] From 1b2418bc2cbba5725f96fb30affb4fc2d833f58a Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 15:11:31 +0200 Subject: [PATCH 2/3] Added detection on unschedulable pods --- pkg/apis/deployment/v1alpha/conditions.go | 2 ++ pkg/deployment/resources/pod_inspector.go | 25 +++++++++++++++++++++-- pkg/util/k8sutil/events.go | 19 +++++++++++++++++ pkg/util/k8sutil/pods.go | 10 +++++++-- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/pkg/apis/deployment/v1alpha/conditions.go b/pkg/apis/deployment/v1alpha/conditions.go index 9ecf645e3..e9d7a12f9 100644 --- a/pkg/apis/deployment/v1alpha/conditions.go +++ b/pkg/apis/deployment/v1alpha/conditions.go @@ -37,6 +37,8 @@ const ( ConditionTypeTerminated ConditionType = "Terminated" // ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once. ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade" + // ConditionTypePodSchedulingFailure indicates that one or more pods belonging to the deployment cannot be schedule. + ConditionTypePodSchedulingFailure ConditionType = "PodSchedulingFailure" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 3cf0963dd..a8b150ae2 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -23,6 +23,7 @@ package resources import ( + "fmt" "time" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" @@ -55,7 +56,8 @@ func (r *Resources) InspectPods() error { // Update member status from all pods found status := r.context.GetStatus() apiObject := r.context.GetAPIObject() - podWithScheduleTimeout := 0 + var podNamesWithScheduleTimeout []string + var unscheduledPodNames []string for _, p := range pods { if k8sutil.IsArangoDBImageIDAndVersionPod(p) { // Image ID pods are not relevant to inspect here @@ -102,7 +104,10 @@ func (r *Resources) InspectPods() error { } if k8sutil.IsPodNotScheduledFor(&p, podScheduleTimeout) { // Pod cannot be scheduled for to long - podWithScheduleTimeout++ + log.Debug().Str("pod-name", p.GetName()).Msg("Pod scheduling timeout") + podNamesWithScheduleTimeout = append(podNamesWithScheduleTimeout, p.GetName()) + } else if !k8sutil.IsPodScheduled(&p) { + unscheduledPodNames = append(unscheduledPodNames, p.GetName()) } if updateMemberStatusNeeded { if err := status.Members.UpdateMemberStatus(memberStatus, group); err != nil { @@ -162,6 +167,22 @@ func (r *Resources) InspectPods() error { // TODO handle other State values } + // Update conditions + if len(podNamesWithScheduleTimeout) > 0 { + if status.Conditions.Update(api.ConditionTypePodSchedulingFailure, true, + "Pods Scheduling Timeout", + fmt.Sprintf("The following pods cannot be scheduled: %v", podNamesWithScheduleTimeout)) { + r.context.CreateEvent(k8sutil.NewPodsSchedulingFailureEvent(podNamesWithScheduleTimeout, r.context.GetAPIObject())) + } + } else if status.Conditions.IsTrue(api.ConditionTypePodSchedulingFailure) && + len(unscheduledPodNames) == 0 { + if status.Conditions.Update(api.ConditionTypePodSchedulingFailure, false, + "Pods Scheduling Resolved", + "No pod reports a scheduling timeout") { + r.context.CreateEvent(k8sutil.NewPodsSchedulingResolvedEvent(r.context.GetAPIObject())) + } + } + // Save status if err := r.context.UpdateStatus(status); err != nil { return maskAny(err) diff --git a/pkg/util/k8sutil/events.go b/pkg/util/k8sutil/events.go index 568264d29..7e4551585 100644 --- a/pkg/util/k8sutil/events.go +++ b/pkg/util/k8sutil/events.go @@ -78,6 +78,25 @@ func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *v1.Event { return event } +// NewPodsSchedulingFailureEvent creates an event indicating that one of more cannot be scheduled. +func NewPodsSchedulingFailureEvent(unscheduledPodNames []string, apiObject APIObject) *v1.Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Pods Scheduling Failure" + event.Message = fmt.Sprintf("One or more pods are not scheduled in time. Pods: %v", unscheduledPodNames) + return event +} + +// NewPodsSchedulingResolvedEvent creates an event indicating that an earlier problem with +// pod scheduling has been resolved. +func NewPodsSchedulingResolvedEvent(apiObject APIObject) *v1.Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Pods Scheduling Resolved" + event.Message = "All pods have been scheduled" + return event +} + // NewErrorEvent creates an even of type error. func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event { event := newDeploymentEvent(apiObject) diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 63a701d3f..1fc53917d 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -88,13 +88,19 @@ func IsPodFailed(pod *v1.Pod) bool { return pod.Status.Phase == v1.PodFailed } +// IsPodScheduled returns true if the pod has been scheduled. +func IsPodScheduled(pod *v1.Pod) bool { + condition := getPodCondition(&pod.Status, v1.PodScheduled) + return condition != nil && condition.Status == v1.ConditionTrue +} + // IsPodNotScheduledFor returns true if the pod has not been scheduled // for longer than the given duration. func IsPodNotScheduledFor(pod *v1.Pod, timeout time.Duration) bool { condition := getPodCondition(&pod.Status, v1.PodScheduled) return condition != nil && - condition.Status == v1.ConditionTrue && - condition.LastTransitionTime.Time.Add(timeout).After(time.Now()) + condition.Status == v1.ConditionFalse && + condition.LastTransitionTime.Time.Add(timeout).Before(time.Now()) } // IsArangoDBImageIDAndVersionPod returns true if the given pod is used for fetching image ID and ArangoDB version of an image From 7f2f400166518dae27524f57fba2417bfdaa2ea0 Mon Sep 17 00:00:00 2001 From: Ewout Prangsma Date: Tue, 27 Mar 2018 15:14:13 +0200 Subject: [PATCH 3/3] Revert example change --- examples/simple-cluster.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/simple-cluster.yaml b/examples/simple-cluster.yaml index 8891a71e8..52fb63621 100644 --- a/examples/simple-cluster.yaml +++ b/examples/simple-cluster.yaml @@ -5,7 +5,6 @@ metadata: spec: mode: cluster image: arangodb/arangodb:3.3.4 - environment: production tls: altNames: ["kube-01", "kube-02", "kube-03"] coordinators: