Skip to content

Commit

Permalink
Merge pull request #2250 from JoelSpeed/mhc-targets
Browse files Browse the repository at this point in the history
✨ Add Health Check logic to MachineHealthCheck Reconciler
  • Loading branch information
k8s-ci-robot authored Feb 28, 2020
2 parents 349f460 + 067f1e0 commit c340b22
Show file tree
Hide file tree
Showing 11 changed files with 1,409 additions and 19 deletions.
9 changes: 7 additions & 2 deletions api/v1alpha3/machinehealthcheck_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ type MachineHealthCheckSpec struct {
// "selector" are not healthy.
// +optional
MaxUnhealthy *intstr.IntOrString `json:"maxUnhealthy,omitempty"`

// Machines older than this duration without a node will be considered to have
// failed and will be remediated.
// +optional
NodeStartupTimeout *metav1.Duration `json:"nodeStartupTimeout,omitempty"`
}

// ANCHOR_END: MachineHealthCHeckSpec
Expand Down Expand Up @@ -73,11 +78,11 @@ type UnhealthyCondition struct {
type MachineHealthCheckStatus struct {
// total number of machines counted by this machine health check
// +kubebuilder:validation:Minimum=0
ExpectedMachines int32 `json:"expectedMachines"`
ExpectedMachines int32 `json:"expectedMachines,omitempty"`

// total number of healthy machines counted by this machine health check
// +kubebuilder:validation:Minimum=0
CurrentHealthy int32 `json:"currentHealthy"`
CurrentHealthy int32 `json:"currentHealthy,omitempty"`
}

// ANCHOR_END: MachineHealthCheckStatus
Expand Down
20 changes: 20 additions & 0 deletions api/v1alpha3/machinehealthcheck_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1alpha3

import (
"fmt"
"time"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -28,6 +29,14 @@ import (
"sigs.k8s.io/controller-runtime/pkg/webhook"
)

var (
// Default time allowed for a node to start up. Can be made longer as part of
// spec if required for particular provider.
// 10 minutes should allow the instance to start and the node to join the
// cluster on most providers.
defaultNodeStartupTimeout = metav1.Duration{Duration: 10 * time.Minute}
)

func (m *MachineHealthCheck) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(m).
Expand All @@ -46,6 +55,10 @@ func (m *MachineHealthCheck) Default() {
defaultMaxUnhealthy := intstr.FromString("100%")
m.Spec.MaxUnhealthy = &defaultMaxUnhealthy
}

if m.Spec.NodeStartupTimeout == nil {
m.Spec.NodeStartupTimeout = &defaultNodeStartupTimeout
}
}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
Expand Down Expand Up @@ -86,6 +99,13 @@ func (m *MachineHealthCheck) validate(old *MachineHealthCheck) error {
)
}

if m.Spec.NodeStartupTimeout != nil && m.Spec.NodeStartupTimeout.Seconds() < 30 {
allErrs = append(
allErrs,
field.Invalid(field.NewPath("spec", "nodeStartupTimeout"), m.Spec.NodeStartupTimeout, "must be at least 30s"),
)
}

if len(allErrs) == 0 {
return nil
}
Expand Down
66 changes: 66 additions & 0 deletions api/v1alpha3/machinehealthcheck_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package v1alpha3

import (
"testing"
"time"

. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -30,6 +31,8 @@ func TestMachineHealthCheckDefault(t *testing.T) {
mhc.Default()

g.Expect(mhc.Spec.MaxUnhealthy.String()).To(Equal("100%"))
g.Expect(mhc.Spec.NodeStartupTimeout).ToNot(BeNil())
g.Expect(*mhc.Spec.NodeStartupTimeout).To(Equal(metav1.Duration{Duration: 10 * time.Minute}))
}

func TestMachineHealthCheckLabelSelectorAsSelectorValidation(t *testing.T) {
Expand Down Expand Up @@ -115,3 +118,66 @@ func TestMachineHealthCheckClusterNameImmutable(t *testing.T) {
})
}
}

func TestMachineHealthCheckNodeStartupTimeout(t *testing.T) {
zero := metav1.Duration{Duration: 0}
twentyNineSeconds := metav1.Duration{Duration: 29 * time.Second}
thirtySeconds := metav1.Duration{Duration: 30 * time.Second}
oneMinute := metav1.Duration{Duration: 1 * time.Minute}
minusOneMinute := metav1.Duration{Duration: -1 * time.Minute}

tests := []struct {
name string
timeout *metav1.Duration
expectErr bool
}{
{
name: "when the nodeStartupTimeout is not given",
timeout: nil,
expectErr: false,
},
{
name: "when the nodeStartupTimeout is greater than 30s",
timeout: &oneMinute,
expectErr: false,
},
{
name: "when the nodeStartupTimeout is 30s",
timeout: &thirtySeconds,
expectErr: false,
},
{
name: "when the nodeStartupTimeout is 29s",
timeout: &twentyNineSeconds,
expectErr: true,
},
{
name: "when the nodeStartupTimeout is less than 0",
timeout: &minusOneMinute,
expectErr: true,
},
{
name: "when the nodeStartupTimeout is 0",
timeout: &zero,
expectErr: true,
},
}

for _, tt := range tests {
g := NewWithT(t)

mhc := &MachineHealthCheck{
Spec: MachineHealthCheckSpec{
NodeStartupTimeout: tt.timeout,
},
}

if tt.expectErr {
g.Expect(mhc.ValidateCreate()).NotTo(Succeed())
g.Expect(mhc.ValidateUpdate(mhc)).NotTo(Succeed())
} else {
g.Expect(mhc.ValidateCreate()).To(Succeed())
g.Expect(mhc.ValidateUpdate(mhc)).To(Succeed())
}
}
}
5 changes: 5 additions & 0 deletions api/v1alpha3/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions config/crd/bases/cluster.x-k8s.io_machinehealthchecks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ spec:
description: Any further remediation is only allowed if at most "MaxUnhealthy"
machines selected by "selector" are not healthy.
x-kubernetes-int-or-string: true
nodeStartupTimeout:
description: Machines older than this duration without a node will
be considered to have failed and will be remediated.
type: string
selector:
description: Label selector to match machines whose health will be
exercised
Expand Down Expand Up @@ -158,9 +162,6 @@ spec:
format: int32
minimum: 0
type: integer
required:
- currentHealthy
- expectedMachines
type: object
type: object
served: true
Expand Down
19 changes: 19 additions & 0 deletions controllers/machine_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"context"

"github.com/pkg/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -47,3 +49,20 @@ func getActiveMachinesInCluster(ctx context.Context, c client.Client, namespace,
}
return machines, nil
}

// hasMatchingLabels verifies that the Label Selector matches the given Labels
func hasMatchingLabels(matchSelector metav1.LabelSelector, matchLabels map[string]string) bool {
// This should never fail, validating webhook should catch this first
selector, err := metav1.LabelSelectorAsSelector(&matchSelector)
if err != nil {
return false
}
// If a nil or empty selector creeps in, it should match nothing, not everything.
if selector.Empty() {
return false
}
if !selector.Matches(labels.Set(matchLabels)) {
return false
}
return true
}
71 changes: 71 additions & 0 deletions controllers/machine_helpers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,3 +127,74 @@ func Test_getActiveMachinesInCluster(t *testing.T) {
})
}
}

func TestMachineHealthCheckHasMatchingLabels(t *testing.T) {
testCases := []struct {
name string
selector metav1.LabelSelector
labels map[string]string
expected bool
}{
{
name: "selector matches labels",

selector: metav1.LabelSelector{
MatchLabels: map[string]string{
"foo": "bar",
},
},

labels: map[string]string{
"foo": "bar",
},

expected: true,
},
{
name: "selector does not match labels",

selector: metav1.LabelSelector{
MatchLabels: map[string]string{
"foo": "bar",
},
},

labels: map[string]string{
"no": "match",
},
expected: false,
},
{
name: "selector is empty",
selector: metav1.LabelSelector{},
labels: map[string]string{},
expected: false,
},
{
name: "seelctor is invalid",
selector: metav1.LabelSelector{
MatchLabels: map[string]string{
"foo": "bar",
},
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Operator: "bad-operator",
},
},
},
labels: map[string]string{
"foo": "bar",
},
expected: false,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
g := NewWithT(t)

got := hasMatchingLabels(tc.selector, tc.labels)
g.Expect(got).To(Equal(tc.expected))
})
}
}
Loading

0 comments on commit c340b22

Please sign in to comment.