⚠️ add docker machine pool and e2e tests for machine pool

exp/util/util.go

@@ -19,11 +19,15 @@ package util
 import (
 	"context"
 
+	"github.com/go-logr/logr"
 	"github.com/pkg/errors"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1alpha3"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 )
 
 // GetOwnerMachinePool returns the MachinePool objects owning the current resource.
@@ -52,3 +56,36 @@ func GetMachinePoolByName(ctx context.Context, c client.Client, namespace, name
 	}
 	return m, nil
 }
+
+// MachinePoolToInfrastructureMapFunc returns a handler.ToRequestsFunc that watches for
+// MachinePool events and returns reconciliation requests for an infrastructure provider object.
+func MachinePoolToInfrastructureMapFunc(gvk schema.GroupVersionKind, log logr.Logger) handler.ToRequestsFunc {
+	log = log.WithValues("machine-pool-to-infra-map-func", gvk.String())
+	return func(o handler.MapObject) []reconcile.Request {
+		log := log.WithValues("namespace", o.Meta.GetNamespace(), "name", o.Meta.GetName())
+		m, ok := o.Object.(*clusterv1exp.MachinePool)
+		if !ok {
+			log.V(4).Info("not a machine pool")
+			return nil
+		}
+
+		gk := gvk.GroupKind()
+		ref := m.Spec.Template.Spec.InfrastructureRef
+		// Return early if the GroupKind doesn't match what we expect.
+		infraGK := ref.GroupVersionKind().GroupKind()
+		if gk != infraGK {
+			log.V(4).Info("infra kind doesn't match filter group kind", infraGK.String())
+			return nil
+		}
+
+		log.V(4).Info("projecting object", "namespace", m.Namespace, "name", ref.Name)
+		return []reconcile.Request{
+			{
+				NamespacedName: client.ObjectKey{
+					Namespace: m.Namespace,
+					Name:      ref.Name,
+				},
+			},
+		}
+	}
+}

test/e2e/Makefile

@@ -59,6 +59,7 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption/step1 --load_restrictor none > $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml
 	echo "---" >> $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml
 	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption/step2 --load_restrictor none >> $(DOCKER_TEMPLATES)/cluster-template-kcp-adoption.yaml
+	$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/cluster-template-machine-pool --load_restrictor none > $(DOCKER_TEMPLATES)/cluster-template-machine-pool.yaml
 
 ## --------------------------------------
 ## Testing

test/e2e/config/docker.yaml

@@ -70,6 +70,7 @@ providers:
   - sourcePath: "../data/infrastructure-docker/cluster-template.yaml"
   - sourcePath: "../data/infrastructure-docker/cluster-template-mhc.yaml"
   - sourcePath: "../data/infrastructure-docker/cluster-template-kcp-adoption.yaml"
+  - sourcePath: "../data/infrastructure-docker/cluster-template-machine-pool.yaml"
 
 variables:
   KUBERNETES_VERSION: "v1.18.2"
@@ -83,12 +84,15 @@ variables:
   DOCKER_POD_CIDRS: "192.168.0.0/16"
   CNI: "./data/cni/kindnet/kindnet.yaml"
   EXP_CLUSTER_RESOURCE_SET: "true"
+  EXP_MACHINE_POOL: "true"
 
 intervals:
   default/wait-controllers: ["3m", "10s"]
   default/wait-cluster: ["5m", "10s"]
   default/wait-control-plane: ["10m", "10s"]
   default/wait-worker-nodes: ["5m", "10s"]
+  default/wait-machine-pool-nodes: ["5m", "10s"]
   default/wait-delete-cluster: ["3m", "10s"]
   default/wait-machine-upgrade: ["20m", "10s"]
+  default/wait-machine-pool-upgrade: ["5m", "10s"]
   default/wait-machine-remediation: ["5m", "10s"]

test/e2e/data/infrastructure-docker/bases/cluster-with-kcp.yaml

@@ -61,7 +61,7 @@ spec:
       controllerManager:
         extraArgs: {enable-hostpath-provisioner: 'true'}
       apiServer:
-        certSANs: [localhost, 127.0.0.1]
+        certSANs: [localhost, 127.0.0.1, 0.0.0.0]
     initConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock

test/e2e/data/infrastructure-docker/bases/mp.yaml

@@ -0,0 +1,39 @@
+---
+# MachinePool which references the DockerMachinePool and KubeadmConfigTemplate below
+apiVersion: exp.cluster.x-k8s.io/v1alpha3
+kind: MachinePool
+metadata:
+  name: "${CLUSTER_NAME}-mp-0"
+spec:
+  clusterName: '${CLUSTER_NAME}'
+  replicas: ${WORKER_MACHINE_COUNT}
+  template:
+    spec:
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+          kind: KubeadmConfig
+          name: "${CLUSTER_NAME}-mp-0-config"
+      clusterName: '${CLUSTER_NAME}'
+      infrastructureRef:
+        apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
+        kind: DockerMachinePool
+        name: "${CLUSTER_NAME}-dmp-0"
+      version: "${KUBERNETES_VERSION}"
+---
+# DockerMachinePool using default values referenced by the MachinePool
+apiVersion: exp.infrastructure.cluster.x-k8s.io/v1alpha3
+kind: DockerMachinePool
+metadata:
+  name: "${CLUSTER_NAME}-dmp-0"
+---
+# KubeadmConfigTemplate referenced by the MachinePool
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha3
+kind: KubeadmConfig
+metadata:
+  name: "${CLUSTER_NAME}-mp-0-config"
+spec:
+  joinConfiguration:
+    nodeRegistration:
+      kubeletExtraArgs:
+        eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

test/e2e/data/infrastructure-docker/cluster-template-machine-pool/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/mp.yaml
+  - ../bases/crs.yaml

test/e2e/kcp_upgrade.go

@@ -27,8 +27,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/pointer"
-	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
-	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
+
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/cluster-api/util"
@@ -46,12 +45,11 @@ type KCPUpgradeSpecInput struct {
 // KCPUpgradeSpec implements a test that verifies KCP to properly upgrade a control plane with 3 machines.
 func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput) {
 	var (
-		specName      = "kcp-upgrade"
-		input         KCPUpgradeSpecInput
-		namespace     *corev1.Namespace
-		cancelWatches context.CancelFunc
-		cluster       *clusterv1.Cluster
-		controlPlane  *controlplanev1.KubeadmControlPlane
+		specName         = "kcp-upgrade"
+		input            KCPUpgradeSpecInput
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
 	)
 
 	BeforeEach(func() {
@@ -73,7 +71,7 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
 	It("Should successfully upgrade Kubernetes, DNS, kube-proxy, and etcd in a single control plane cluster", func() {
 
 		By("Creating a workload cluster")
-		cluster, controlPlane, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+		clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: input.BootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
 				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
@@ -95,8 +93,8 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
 		By("Upgrading Kubernetes, DNS, kube-proxy, and etcd versions")
 		framework.UpgradeControlPlaneAndWaitForUpgrade(ctx, framework.UpgradeControlPlaneAndWaitForUpgradeInput{
 			ClusterProxy:                input.BootstrapClusterProxy,
-			Cluster:                     cluster,
-			ControlPlane:                controlPlane,
+			Cluster:                     clusterResources.Cluster,
+			ControlPlane:                clusterResources.ControlPlane,
 			EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
 			DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
 			KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
@@ -112,7 +110,7 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
 
 		By("Creating a workload cluster")
 
-		cluster, controlPlane, _ = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+		clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
 			ClusterProxy: input.BootstrapClusterProxy,
 			ConfigCluster: clusterctl.ConfigClusterInput{
 				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
@@ -134,8 +132,8 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
 		By("Upgrading Kubernetes, DNS, kube-proxy, and etcd versions")
 		framework.UpgradeControlPlaneAndWaitForUpgrade(ctx, framework.UpgradeControlPlaneAndWaitForUpgradeInput{
 			ClusterProxy:                input.BootstrapClusterProxy,
-			Cluster:                     cluster,
-			ControlPlane:                controlPlane,
+			Cluster:                     clusterResources.Cluster,
+			ControlPlane:                clusterResources.ControlPlane,
 			EtcdImageTag:                input.E2EConfig.GetVariable(EtcdVersionUpgradeTo),
 			DNSImageTag:                 input.E2EConfig.GetVariable(CoreDNSVersionUpgradeTo),
 			KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
@@ -149,6 +147,6 @@ func KCPUpgradeSpec(ctx context.Context, inputGetter func() KCPUpgradeSpecInput)
 
 	AfterEach(func() {
 		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
-		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
 	})
 }

test/e2e/machine_pool.go

@@ -0,0 +1,127 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// MachinePoolInput is the input for MachinePoolSpec
+type MachinePoolInput struct {
+	E2EConfig             *clusterctl.E2EConfig
+	ClusterctlConfigPath  string
+	BootstrapClusterProxy framework.ClusterProxy
+	ArtifactFolder        string
+	SkipCleanup           bool
+}
+
+// MachinePoolSpec implements a test that verifies MachinePool scale up, down and version update
+func MachinePoolSpec(ctx context.Context, inputGetter func() MachinePoolInput) {
+	var (
+		specName         = "machine-pool"
+		input            MachinePoolInput
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0755)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeFrom))
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+	})
+
+	It("Should successfully create a cluster with machine pool machines", func() {
+		By("Creating a workload cluster")
+		workerMachineCount := int32(2)
+		clusterResources = clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+			ClusterProxy: input.BootstrapClusterProxy,
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+				Flavor:                   "machine-pool",
+				Namespace:                namespace.Name,
+				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
+				ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				WorkerMachineCount:       pointer.Int64Ptr(int64(workerMachineCount)),
+			},
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachinePools:          input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
+		})
+
+		By("Scaling the machine pool up")
+		framework.ScaleMachinePoolAndWait(context.TODO(), framework.ScaleMachinePoolAndWaitInput{
+			ClusterProxy:              input.BootstrapClusterProxy,
+			Cluster:                   clusterResources.Cluster,
+			Replicas:                  workerMachineCount + 1,
+			MachinePools:              clusterResources.MachinePools,
+			WaitForMachinePoolToScale: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
+		})
+
+		By("Scaling the machine pool down")
+		framework.ScaleMachinePoolAndWait(context.TODO(), framework.ScaleMachinePoolAndWaitInput{
+			ClusterProxy:              input.BootstrapClusterProxy,
+			Cluster:                   clusterResources.Cluster,
+			Replicas:                  workerMachineCount - 1,
+			MachinePools:              clusterResources.MachinePools,
+			WaitForMachinePoolToScale: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
+		})
+
+		By("Upgrading the instances")
+		framework.UpgradeMachinePoolAndWait(context.TODO(), framework.UpgradeMachinePoolAndWaitInput{
+			ClusterProxy:                   input.BootstrapClusterProxy,
+			Cluster:                        clusterResources.Cluster,
+			UpgradeVersion:                 input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			WaitForMachinePoolToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-upgrade"),
+			MachinePools:                   clusterResources.MachinePools,
+		})
+
+		By("PASSED!")
+	})
+
+	AfterEach(func() {
+		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+	})
+}

test/e2e/machine_pool_test.go

@@ -0,0 +1,37 @@
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo"
+)
+
+var _ = Describe("When testing MachinePools", func() {
+	MachinePoolSpec(context.TODO(), func() MachinePoolInput {
+		return MachinePoolInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+		}
+	})
+})