Skip to content

Commit

Permalink
[e2e] add mhc test
Browse files Browse the repository at this point in the history
  • Loading branch information
Sedef committed May 11, 2020
1 parent bfa4990 commit 1741057
Show file tree
Hide file tree
Showing 7 changed files with 382 additions and 3 deletions.
2 changes: 1 addition & 1 deletion test/e2e/config/docker-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,4 @@ intervals:
default/wait-worker-nodes: ["5m", "10s"]
default/wait-delete-cluster: ["3m", "10s"]
default/wait-machine-upgrade: ["20m", "10s"]
default/wait-machine-remediation: ["3m", "10s"]
default/wait-machine-remediation: ["5m", "10s"]
18 changes: 18 additions & 0 deletions test/e2e/data/infrastructure-docker/cluster-template-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ spec:
selector:
matchLabels:
template:
metadata:
labels:
"nodepool": "pool1"
spec:
clusterName: "${ CLUSTER_NAME }"
version: "${ KUBERNETES_VERSION }"
Expand All @@ -105,3 +108,18 @@ spec:
name: "${ CLUSTER_NAME }-md-0"
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerMachineTemplate
---
apiVersion: cluster.x-k8s.io/v1alpha3
kind: MachineHealthCheck
metadata:
name: "${CLUSTER_NAME}-mhc-0"
spec:
clusterName: "${ CLUSTER_NAME }"
maxUnhealthy: 100%
selector:
matchLabels:
nodepool: "pool1"
unhealthyConditions:
- type: E2ENodeUnhealthy
status: "True"
timeout: 30s
18 changes: 18 additions & 0 deletions test/e2e/data/infrastructure-docker/cluster-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ spec:
selector:
matchLabels:
template:
metadata:
labels:
"nodepool": "pool1"
spec:
clusterName: "${ CLUSTER_NAME }"
version: "${ KUBERNETES_VERSION }"
Expand All @@ -105,3 +108,18 @@ spec:
name: "${ CLUSTER_NAME }-md-0"
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha3
kind: DockerMachineTemplate
---
apiVersion: cluster.x-k8s.io/v1alpha3
kind: MachineHealthCheck
metadata:
name: "${CLUSTER_NAME}-mhc-0"
spec:
clusterName: "${ CLUSTER_NAME }"
maxUnhealthy: 100%
selector:
matchLabels:
nodepool: "pool1"
unhealthyConditions:
- type: E2ENodeUnhealthy
status: "True"
timeout: 30s
108 changes: 108 additions & 0 deletions test/e2e/mhc_remediations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"
"fmt"
"os"
"path/filepath"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

corev1 "k8s.io/api/core/v1"
"k8s.io/utils/pointer"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/cluster-api/util"
)

// MachineRemediationSpecInput is the input for MachineRemediationSpec.
type MachineRemediationSpecInput struct {
E2EConfig *clusterctl.E2EConfig
ClusterctlConfigPath string
BootstrapClusterProxy framework.ClusterProxy
ArtifactFolder string
SkipCleanup bool
}

// MachineRemediationSpec implements a test that verifies that Machines are remediated by MHC during unhealthy conditions.
func MachineRemediationSpec(ctx context.Context, inputGetter func() MachineRemediationSpecInput) {
var (
specName = "mhc-remediation"
input MachineRemediationSpecInput
namespace *corev1.Namespace
cancelWatches context.CancelFunc
cluster *clusterv1.Cluster
)

BeforeEach(func() {
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
input = inputGetter()
Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
Expect(input.E2EConfig.Variables).To(HaveKey(CNIPath))

// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
})

It("Should successfully remediate unhealthy machines with MachineHealthCheck", func() {

By("Creating a workload cluster")

var mds []*clusterv1.MachineDeployment
cluster, _, mds = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
ClusterProxy: input.BootstrapClusterProxy,
ConfigCluster: clusterctl.ConfigClusterInput{
LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
ClusterctlConfigPath: input.ClusterctlConfigPath,
KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(),
InfrastructureProvider: clusterctl.DefaultInfrastructureProvider,
Flavor: clusterctl.DefaultFlavor,
Namespace: namespace.Name,
ClusterName: fmt.Sprintf("cluster-%s", util.RandomString(6)),
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion),
ControlPlaneMachineCount: pointer.Int64Ptr(1),
WorkerMachineCount: pointer.Int64Ptr(1),
},
CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"),
WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
})

By("Waiting for MachineHealthCheck remediation")
framework.DiscoverMachineHealthChecksAndWaitForRemediation(ctx, framework.DiscoverMachineHealthCheckAndWaitForRemediationInput{
ClusterProxy: input.BootstrapClusterProxy,
Cluster: cluster,
WaitForMachineRemediation: input.E2EConfig.GetIntervals(specName, "wait-machine-remediation"),
})

By("PASSED!")
})

AfterEach(func() {
// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
})
}
39 changes: 39 additions & 0 deletions test/e2e/mhc_remediations_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// +build e2e

/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
"context"

. "github.com/onsi/ginkgo"
)

var _ = Describe("When testing unhealthy machines remediation", func() {

MachineRemediationSpec(context.TODO(), func() MachineRemediationSpecInput {
return MachineRemediationSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
}
})

})
27 changes: 25 additions & 2 deletions test/framework/machine_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,29 @@ func GetMachinesByMachineDeployments(ctx context.Context, input GetMachinesByMac
return machineList.Items
}

// GetMachinesByMachineHealthCheckInput is the input for GetMachinesByMachineHealthCheck.
type GetMachinesByMachineHealthCheckInput struct {
Lister Lister
ClusterName string
MachineHealthCheck *clusterv1.MachineHealthCheck
}

// GetMachinesByMachineHealthCheckInput returns Machine objects for a cluster that match with MachineHealthCheck selector.
func GetMachinesByMachineHealthCheck(ctx context.Context, input GetMachinesByMachineHealthCheckInput) []clusterv1.Machine {
Expect(ctx).NotTo(BeNil(), "ctx is required for GetMachinesByMachineDeployments")
Expect(input.Lister).ToNot(BeNil(), "Invalid argument. input.Lister can't be nil when calling GetMachinesByMachineHealthCheck")
Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling GetMachinesByMachineHealthCheck")
Expect(input.MachineHealthCheck).ToNot(BeNil(), "Invalid argument. input.MachineHealthCheck can't be nil when calling GetMachinesByMachineHealthCheck")

opts := byClusterOptions(input.ClusterName, input.MachineHealthCheck.Namespace)
opts = append(opts, machineHealthCheckOptions(*input.MachineHealthCheck)...)

machineList := &clusterv1.MachineList{}
Expect(input.Lister.List(ctx, machineList, opts...)).To(Succeed(), "Failed to list MachineList object for Cluster %s/%s", input.MachineHealthCheck.Namespace, input.ClusterName)

return machineList.Items
}

// GetControlPlaneMachinesByClusterInput is the input for GetControlPlaneMachinesByCluster.
type GetControlPlaneMachinesByClusterInput struct {
Lister Lister
Expand Down Expand Up @@ -100,7 +123,7 @@ func WaitForControlPlaneMachinesToBeUpgraded(ctx context.Context, input WaitForC
fmt.Fprintf(GinkgoWriter, "Ensuring all MachineDeployment Machines have upgraded kubernetes version %s\n", input.KubernetesUpgradeVersion)

Eventually(func() (int, error) {
machines := GetControlPlaneMachinesByCluster(context.TODO(), GetControlPlaneMachinesByClusterInput{
machines := GetControlPlaneMachinesByCluster(ctx, GetControlPlaneMachinesByClusterInput{
Lister: input.Lister,
ClusterName: input.Cluster.Name,
Namespace: input.Cluster.Namespace,
Expand Down Expand Up @@ -139,7 +162,7 @@ func WaitForMachineDeploymentMachinesToBeUpgraded(ctx context.Context, input Wai

fmt.Fprintf(GinkgoWriter, "Ensuring all MachineDeployment Machines have upgraded kubernetes version %s\n", input.KubernetesUpgradeVersion)
Eventually(func() (int, error) {
machines := GetMachinesByMachineDeployments(context.TODO(), GetMachinesByMachineDeploymentsInput{
machines := GetMachinesByMachineDeployments(ctx, GetMachinesByMachineDeploymentsInput{
Lister: input.Lister,
ClusterName: input.Cluster.Name,
Namespace: input.Cluster.Namespace,
Expand Down
Loading

0 comments on commit 1741057

Please sign in to comment.