From fbb97871c2fae9f3b4b4848b97385f50b536aea3 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Mon, 11 Mar 2019 12:05:30 -0400 Subject: [PATCH] Add pivot phase for clusterctl (#731) - Add `clusterctl alpha phases pivot` subcommand - Pivot ClusterAPI objects for all namespaces - Adds MachineClass support to pivoting - Unify pivoting approach - Scale down StatefulSets on pivot to avoid dueling controllers - Remove ownerRefs prior to pivoting objects - Change order of pivoting of Machine* objects to avoid race conditions - Recursively copy objects based on ownerRefs - `clusterctl delete cluster` now pivots and deletes Cluster API resources from all namespaces. Co-authored-by: Chuck Ha Signed-off-by: Jason DeTiberus --- Gopkg.lock | 2 + cmd/clusterctl/clusterdeployer/BUILD.bazel | 1 - .../clusterdeployer/clusterclient/BUILD.bazel | 1 + .../clusterclient/clusterclient.go | 405 +++++++-- .../clusterdeployer/clusterdeployer.go | 154 +--- .../clusterdeployer/clusterdeployer_test.go | 860 +++++++++++------- cmd/clusterctl/cmd/BUILD.bazel | 1 + cmd/clusterctl/cmd/alpha_phase_pivot.go | 100 ++ cmd/clusterctl/cmd/delete_cluster.go | 6 +- cmd/clusterctl/phases/BUILD.bazel | 15 +- cmd/clusterctl/phases/applymachines.go | 2 +- cmd/clusterctl/phases/pivot.go | 424 +++++++++ cmd/clusterctl/phases/pivot_test.go | 543 +++++++++++ ...delete-cluster-no-args-invalid-flag.golden | 1 - .../testdata/delete-cluster-no-args.golden | 1 - config/default/manager_image_patch.yaml | 2 +- 16 files changed, 1972 insertions(+), 546 deletions(-) create mode 100644 cmd/clusterctl/cmd/alpha_phase_pivot.go create mode 100644 cmd/clusterctl/phases/pivot.go create mode 100644 cmd/clusterctl/phases/pivot_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 0abd842c8d01..873df1d65cf3 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1132,6 +1132,8 @@ "github.com/spf13/cobra", "github.com/spf13/pflag", "golang.org/x/net/context", + "k8s.io/api/apps/v1", + "k8s.io/api/autoscaling/v1", "k8s.io/api/core/v1", "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1", "k8s.io/apimachinery/pkg/api/equality", diff --git a/cmd/clusterctl/clusterdeployer/BUILD.bazel b/cmd/clusterctl/clusterdeployer/BUILD.bazel index c8bbe34bce0a..0fdbfaafc620 100644 --- a/cmd/clusterctl/clusterdeployer/BUILD.bazel +++ b/cmd/clusterctl/clusterdeployer/BUILD.bazel @@ -15,7 +15,6 @@ go_library( "//cmd/clusterctl/phases:go_default_library", "//cmd/clusterctl/providercomponents:go_default_library", "//pkg/apis/cluster/v1alpha1:go_default_library", - "//pkg/util:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/cmd/clusterctl/clusterdeployer/clusterclient/BUILD.bazel b/cmd/clusterctl/clusterdeployer/clusterclient/BUILD.bazel index dcfccba82ce4..69dad4597633 100644 --- a/cmd/clusterctl/clusterdeployer/clusterclient/BUILD.bazel +++ b/cmd/clusterctl/clusterdeployer/clusterclient/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//pkg/client/clientset_generated/clientset:go_default_library", "//pkg/util:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/api/autoscaling/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", diff --git a/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go b/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go index 12681a35e182..d0aa91dcd88c 100644 --- a/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go +++ b/cmd/clusterctl/clusterdeployer/clusterclient/clusterclient.go @@ -17,6 +17,7 @@ limitations under the License. package clusterclient import ( + "fmt" "io/ioutil" "net" "os" @@ -27,6 +28,7 @@ import ( "time" "github.com/pkg/errors" + autoscalingv1 "k8s.io/api/autoscaling/v1" apiv1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,38 +50,48 @@ const ( timeoutResourceReady = 15 * time.Minute timeoutMachineReady = 30 * time.Minute timeoutResourceDelete = 15 * time.Minute + machineClusterLabelName = "cluster.k8s.io/cluster-name" ) // Provides interaction with a cluster type Client interface { - GetContextNamespace() string Apply(string) error + Close() error + CreateClusterObject(*clusterv1.Cluster) error + CreateMachineClass(*clusterv1.MachineClass) error + CreateMachineDeployments([]*clusterv1.MachineDeployment, string) error + CreateMachineSets([]*clusterv1.MachineSet, string) error + CreateMachines([]*clusterv1.Machine, string) error Delete(string) error + DeleteClusters(string) error + DeleteNamespace(string) error + DeleteMachineClasses(string) error + DeleteMachineClass(namespace, name string) error + DeleteMachineDeployments(string) error + DeleteMachineSets(string) error + DeleteMachines(string) error + ForceDeleteCluster(namespace, name string) error + ForceDeleteMachine(namespace, name string) error + ForceDeleteMachineSet(namespace, name string) error + ForceDeleteMachineDeployment(namespace, name string) error + EnsureNamespace(string) error + GetClusters(string) ([]*clusterv1.Cluster, error) + GetCluster(string, string) (*clusterv1.Cluster, error) + GetContextNamespace() string + GetMachineClasses(namespace string) ([]*clusterv1.MachineClass, error) + GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error) + GetMachineDeploymentsForCluster(*clusterv1.Cluster) ([]*clusterv1.MachineDeployment, error) + GetMachineDeployments(string) ([]*clusterv1.MachineDeployment, error) + GetMachineSet(namespace, name string) (*clusterv1.MachineSet, error) + GetMachineSets(namespace string) ([]*clusterv1.MachineSet, error) + GetMachineSetsForCluster(*clusterv1.Cluster) ([]*clusterv1.MachineSet, error) + GetMachineSetsForMachineDeployment(*clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) + GetMachines(namespace string) ([]*clusterv1.Machine, error) + GetMachinesForCluster(*clusterv1.Cluster) ([]*clusterv1.Machine, error) + GetMachinesForMachineSet(*clusterv1.MachineSet) ([]*clusterv1.Machine, error) + ScaleStatefulSet(namespace, name string, scale int32) error WaitForClusterV1alpha1Ready() error - GetClusterObjectsInNamespace(string) ([]*clusterv1.Cluster, error) - GetClusterObject(string, string) (*clusterv1.Cluster, error) - GetMachineDeploymentObjects() ([]*clusterv1.MachineDeployment, error) - GetMachineDeploymentObjectsInNamespace(string) ([]*clusterv1.MachineDeployment, error) - GetMachineSetObjects() ([]*clusterv1.MachineSet, error) - GetMachineSetObjectsInNamespace(string) ([]*clusterv1.MachineSet, error) - GetMachineObjects() ([]*clusterv1.Machine, error) - GetMachineObjectsInNamespace(ns string) ([]*clusterv1.Machine, error) - CreateClusterObject(*clusterv1.Cluster) error - CreateMachineDeploymentObjects([]*clusterv1.MachineDeployment, string) error - CreateMachineSetObjects([]*clusterv1.MachineSet, string) error - CreateMachineObjects([]*clusterv1.Machine, string) error - DeleteClusterObjectsInNamespace(string) error - DeleteClusterObjects() error - DeleteMachineDeploymentObjectsInNamespace(string) error - DeleteMachineDeploymentObjects() error - DeleteMachineSetObjectsInNamespace(string) error - DeleteMachineSetObjects() error - DeleteMachineObjectsInNamespace(string) error - DeleteMachineObjects() error UpdateClusterObjectEndpoint(string, string, string) error - EnsureNamespace(string) error - DeleteNamespace(string) error - Close() error } type client struct { @@ -127,6 +139,27 @@ func (c *client) EnsureNamespace(namespaceName string) error { return nil } +func (c *client) ScaleStatefulSet(ns string, name string, scale int32) error { + clientset, err := clientcmd.NewCoreClientSetForDefaultSearchPath(c.kubeconfigFile, clientcmd.NewConfigOverrides()) + if err != nil { + return errors.Wrap(err, "error creating core clientset") + } + + _, err = clientset.AppsV1().StatefulSets(ns).UpdateScale(name, &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: scale, + }, + }) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + func (c *client) DeleteNamespace(namespaceName string) error { if namespaceName == apiv1.NamespaceDefault { return nil @@ -181,8 +214,8 @@ func (c *client) GetContextNamespace() string { return c.configOverrides.Context.Namespace } -func (c *client) GetClusterObject(name, ns string) (*clusterv1.Cluster, error) { - clustersInNamespace, err := c.GetClusterObjectsInNamespace(ns) +func (c *client) GetCluster(name, ns string) (*clusterv1.Cluster, error) { + clustersInNamespace, err := c.GetClusters(ns) if err != nil { return nil, err } @@ -196,7 +229,27 @@ func (c *client) GetClusterObject(name, ns string) (*clusterv1.Cluster, error) { return cluster, nil } -func (c *client) GetClusterObjectsInNamespace(namespace string) ([]*clusterv1.Cluster, error) { +// ForceDeleteCluster removes the finalizer for a Cluster prior to deleting, this is used during pivot +func (c *client) ForceDeleteCluster(namespace, name string) error { + cluster, err := c.clientSet.ClusterV1alpha1().Clusters(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting cluster %s/%s", namespace, name) + } + + cluster.ObjectMeta.SetFinalizers([]string{}) + + if _, err := c.clientSet.ClusterV1alpha1().Clusters(namespace).Update(cluster); err != nil { + return errors.Wrapf(err, "error removing finalizer on cluster %s/%s", namespace, name) + } + + if err := c.clientSet.ClusterV1alpha1().Clusters(namespace).Delete(name, &metav1.DeleteOptions{}); err != nil { + return errors.Wrapf(err, "error deleting cluster %s/%s", namespace, name) + } + + return nil +} + +func (c *client) GetClusters(namespace string) ([]*clusterv1.Cluster, error) { clusters := []*clusterv1.Cluster{} clusterlist, err := c.clientSet.ClusterV1alpha1().Clusters(namespace).List(metav1.ListOptions{}) if err != nil { @@ -209,7 +262,46 @@ func (c *client) GetClusterObjectsInNamespace(namespace string) ([]*clusterv1.Cl return clusters, nil } -func (c *client) GetMachineDeploymentObjectsInNamespace(namespace string) ([]*clusterv1.MachineDeployment, error) { +func (c *client) GetMachineClasses(namespace string) ([]*clusterv1.MachineClass, error) { + machineClassesList, err := c.clientSet.ClusterV1alpha1().MachineClasses(namespace).List(metav1.ListOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error listing MachineClasses in namespace %q", namespace) + } + var machineClasses []*clusterv1.MachineClass + for i := 0; i < len(machineClassesList.Items); i++ { + machineClasses = append(machineClasses, &machineClassesList.Items[i]) + } + return machineClasses, nil +} + +func (c *client) GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error) { + machineDeployment, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting MachineDeployment: %s/%s", namespace, name) + } + return machineDeployment, nil +} + +func (c *client) GetMachineDeploymentsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineDeployment, error) { + machineDeploymentList, err := c.clientSet.ClusterV1alpha1().MachineDeployments(cluster.Namespace).List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", machineClusterLabelName, cluster.Name), + }) + if err != nil { + return nil, errors.Wrapf(err, "error listing MachineDeployments for Cluster %s/%s", cluster.Namespace, cluster.Name) + } + var machineDeployments []*clusterv1.MachineDeployment + for _, md := range machineDeploymentList.Items { + for _, or := range md.GetOwnerReferences() { + if or.Kind == cluster.Kind && or.Name == cluster.Name { + machineDeployments = append(machineDeployments, &md) + continue + } + } + } + return machineDeployments, nil +} + +func (c *client) GetMachineDeployments(namespace string) ([]*clusterv1.MachineDeployment, error) { machineDeploymentList, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).List(metav1.ListOptions{}) if err != nil { return nil, errors.Wrapf(err, "error listing machine deployment objects in namespace %q", namespace) @@ -221,16 +313,18 @@ func (c *client) GetMachineDeploymentObjectsInNamespace(namespace string) ([]*cl return machineDeployments, nil } -// Deprecated API. Please do not extend or use. -func (c *client) GetMachineDeploymentObjects() ([]*clusterv1.MachineDeployment, error) { - klog.V(2).Info("GetMachineDeploymentObjects API is deprecated, use GetMachineDeploymentObjectsInNamespace instead") - return c.GetMachineDeploymentObjectsInNamespace(apiv1.NamespaceDefault) +func (c *client) GetMachineSet(namespace, name string) (*clusterv1.MachineSet, error) { + machineSet, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting MachineSet: %s/%s", namespace, name) + } + return machineSet, nil } -func (c *client) GetMachineSetObjectsInNamespace(namespace string) ([]*clusterv1.MachineSet, error) { +func (c *client) GetMachineSets(namespace string) ([]*clusterv1.MachineSet, error) { machineSetList, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).List(metav1.ListOptions{}) if err != nil { - return nil, errors.Wrapf(err, "error listing machine set objects in namespace %q", namespace) + return nil, errors.Wrapf(err, "error listing MachineSets in namespace %q", namespace) } var machineSets []*clusterv1.MachineSet for i := 0; i < len(machineSetList.Items); i++ { @@ -239,17 +333,44 @@ func (c *client) GetMachineSetObjectsInNamespace(namespace string) ([]*clusterv1 return machineSets, nil } -// Deprecated API. Please do not extend or use. -func (c *client) GetMachineSetObjects() ([]*clusterv1.MachineSet, error) { - klog.V(2).Info("GetMachineSetObjects API is deprecated, use GetMachineSetObjectsInNamespace instead") - return c.GetMachineSetObjectsInNamespace(apiv1.NamespaceDefault) +func (c *client) GetMachineSetsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineSet, error) { + machineSetList, err := c.clientSet.ClusterV1alpha1().MachineSets(cluster.Namespace).List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", machineClusterLabelName, cluster.Name), + }) + if err != nil { + return nil, errors.Wrapf(err, "error listing MachineSets for Cluster %s/%s", cluster.Namespace, cluster.Name) + } + var machineSets []*clusterv1.MachineSet + for _, ms := range machineSetList.Items { + for _, or := range ms.GetOwnerReferences() { + if or.Kind == cluster.Kind && or.Name == cluster.Name { + machineSets = append(machineSets, &ms) + continue + } + } + } + return machineSets, nil +} + +func (c *client) GetMachineSetsForMachineDeployment(md *clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) { + machineSets, err := c.GetMachineSets(md.Namespace) + if err != nil { + return nil, err + } + var controlledMachineSets []*clusterv1.MachineSet + for _, ms := range machineSets { + if metav1.GetControllerOf(ms) != nil && metav1.IsControlledBy(ms, md) { + controlledMachineSets = append(controlledMachineSets, ms) + } + } + return controlledMachineSets, nil } -func (c *client) GetMachineObjectsInNamespace(namespace string) ([]*clusterv1.Machine, error) { +func (c *client) GetMachines(namespace string) ([]*clusterv1.Machine, error) { machines := []*clusterv1.Machine{} machineslist, err := c.clientSet.ClusterV1alpha1().Machines(namespace).List(metav1.ListOptions{}) if err != nil { - return nil, errors.Wrapf(err, "error listing machine objects in namespace %q", namespace) + return nil, errors.Wrapf(err, "error listing Machines in namespace %q", namespace) } for i := 0; i < len(machineslist.Items); i++ { @@ -258,10 +379,55 @@ func (c *client) GetMachineObjectsInNamespace(namespace string) ([]*clusterv1.Ma return machines, nil } -// Deprecated API. Please do not extend or use. -func (c *client) GetMachineObjects() ([]*clusterv1.Machine, error) { - klog.V(2).Info("GetMachineObjects API is deprecated, use GetMachineObjectsInNamespace instead") - return c.GetMachineObjectsInNamespace(apiv1.NamespaceDefault) +func (c *client) GetMachinesForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.Machine, error) { + machineslist, err := c.clientSet.ClusterV1alpha1().Machines(cluster.Namespace).List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", machineClusterLabelName, cluster.Name), + }) + if err != nil { + return nil, errors.Wrapf(err, "error listing Machines for Cluster %s/%s", cluster.Namespace, cluster.Name) + } + var machines []*clusterv1.Machine + for _, m := range machineslist.Items { + for _, or := range m.GetOwnerReferences() { + if or.Kind == cluster.Kind && or.Name == cluster.Name { + machines = append(machines, &m) + continue + } + } + } + for i := 0; i < len(machineslist.Items); i++ { + machines = append(machines, &machineslist.Items[i]) + } + return machines, nil +} + +func (c *client) GetMachinesForMachineSet(ms *clusterv1.MachineSet) ([]*clusterv1.Machine, error) { + machines, err := c.GetMachines(ms.Namespace) + if err != nil { + return nil, err + } + var controlledMachines []*clusterv1.Machine + for _, m := range machines { + if metav1.GetControllerOf(m) != nil && metav1.IsControlledBy(m, ms) { + controlledMachines = append(controlledMachines, m) + } + } + return controlledMachines, nil +} + +func (c *client) CreateMachineClass(machineClass *clusterv1.MachineClass) error { + _, err := c.clientSet.ClusterV1alpha1().MachineClasses(machineClass.Namespace).Create(machineClass) + if err != nil { + return errors.Wrapf(err, "error creating MachineClass %s/%s", machineClass.Namespace, machineClass.Name) + } + return nil +} + +func (c *client) DeleteMachineClass(namespace, name string) error { + if err := c.clientSet.ClusterV1alpha1().MachineClasses(namespace).Delete(name, newDeleteOptions()); err != nil { + return errors.Wrapf(err, "error deleting MachineClass %s/%s", namespace, name) + } + return nil } func (c *client) CreateClusterObject(cluster *clusterv1.Cluster) error { @@ -274,10 +440,10 @@ func (c *client) CreateClusterObject(cluster *clusterv1.Cluster) error { if err != nil { return errors.Wrapf(err, "error creating cluster in namespace %v", namespace) } - return err + return nil } -func (c *client) CreateMachineDeploymentObjects(deployments []*clusterv1.MachineDeployment, namespace string) error { +func (c *client) CreateMachineDeployments(deployments []*clusterv1.MachineDeployment, namespace string) error { for _, deploy := range deployments { // TODO: Run in parallel https://github.com/kubernetes-sigs/cluster-api/issues/258 _, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).Create(deploy) @@ -288,7 +454,7 @@ func (c *client) CreateMachineDeploymentObjects(deployments []*clusterv1.Machine return nil } -func (c *client) CreateMachineSetObjects(machineSets []*clusterv1.MachineSet, namespace string) error { +func (c *client) CreateMachineSets(machineSets []*clusterv1.MachineSet, namespace string) error { for _, ms := range machineSets { // TODO: Run in parallel https://github.com/kubernetes-sigs/cluster-api/issues/258 _, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).Create(ms) @@ -299,7 +465,7 @@ func (c *client) CreateMachineSetObjects(machineSets []*clusterv1.MachineSet, na return nil } -func (c *client) CreateMachineObjects(machines []*clusterv1.Machine, namespace string) error { +func (c *client) CreateMachines(machines []*clusterv1.Machine, namespace string) error { var ( wg sync.WaitGroup errOnce sync.Once @@ -329,74 +495,107 @@ func (c *client) CreateMachineObjects(machines []*clusterv1.Machine, namespace s return gerr } -// Deprecated API. Please do not extend or use. -func (c *client) DeleteClusterObjects() error { - klog.V(2).Info("DeleteClusterObjects API is deprecated, use DeleteClusterObjectsInNamespace instead") - return c.DeleteClusterObjectsInNamespace(apiv1.NamespaceDefault) -} - -func (c *client) DeleteClusterObjectsInNamespace(namespace string) error { +func (c *client) DeleteClusters(namespace string) error { err := c.clientSet.ClusterV1alpha1().Clusters(namespace).DeleteCollection(newDeleteOptions(), metav1.ListOptions{}) if err != nil { - return errors.Wrapf(err, "error deleting cluster objects in namespace %q", namespace) + return errors.Wrapf(err, "error deleting Clusters in namespace %q", namespace) } err = c.waitForClusterDelete(namespace) if err != nil { - return errors.Wrapf(err, "error waiting for cluster(s) deletion to complete in namespace %q", namespace) + return errors.Wrapf(err, "error waiting for Cluster(s) deletion to complete in namespace %q", namespace) } return nil } -// Deprecated API. Please do not extend or use. -func (c *client) DeleteMachineDeploymentObjects() error { - klog.V(2).Info("DeleteMachineDeploymentObjects API is deprecated, use DeleteMachineDeploymentObjectsInNamespace instead") - return c.DeleteMachineDeploymentObjectsInNamespace(apiv1.NamespaceDefault) +func (c *client) DeleteMachineClasses(namespace string) error { + err := c.clientSet.ClusterV1alpha1().MachineClasses(namespace).DeleteCollection(newDeleteOptions(), metav1.ListOptions{}) + if err != nil { + return errors.Wrapf(err, "error deleting MachineClasses in namespace %q", namespace) + } + err = c.waitForMachineClassesDelete(namespace) + if err != nil { + return errors.Wrapf(err, "error waiting for MachineClass(es) deletion to complete in namespace %q", namespace) + } + return nil } -func (c *client) DeleteMachineDeploymentObjectsInNamespace(namespace string) error { +func (c *client) DeleteMachineDeployments(namespace string) error { err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).DeleteCollection(newDeleteOptions(), metav1.ListOptions{}) if err != nil { - return errors.Wrapf(err, "error deleting machine deployment objects in namespace %q", namespace) + return errors.Wrapf(err, "error deleting MachineDeployments in namespace %q", namespace) } err = c.waitForMachineDeploymentsDelete(namespace) if err != nil { - return errors.Wrapf(err, "error waiting for machine deployment(s) deletion to complete in namespace %q", namespace) + return errors.Wrapf(err, "error waiting for MachineDeployment(s) deletion to complete in namespace %q", namespace) } return nil } -// Deprecated API. Please do not extend or use. -func (c *client) DeleteMachineSetObjects() error { - klog.V(2).Info("DeleteMachineSetObjects API is deprecated, use DeleteMachineSetObjectsInNamespace instead") - return c.DeleteMachineSetObjectsInNamespace(apiv1.NamespaceDefault) -} - -func (c *client) DeleteMachineSetObjectsInNamespace(namespace string) error { +func (c *client) DeleteMachineSets(namespace string) error { err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).DeleteCollection(newDeleteOptions(), metav1.ListOptions{}) if err != nil { - return errors.Wrapf(err, "error deleting machine set objects in namespace %q", namespace) + return errors.Wrapf(err, "error deleting MachineSets in namespace %q", namespace) } err = c.waitForMachineSetsDelete(namespace) if err != nil { - return errors.Wrapf(err, "error waiting for machine set(s) deletion to complete in namespace %q", namespace) + return errors.Wrapf(err, "error waiting for MachineSet(s) deletion to complete in namespace %q", namespace) } return nil } -// Deprecated API. Please do not extend or use. -func (c *client) DeleteMachineObjects() error { - klog.V(2).Info("DeleteMachineObjects API is deprecated, use DeleteMachineObjectsInNamespace instead") - return c.DeleteMachineObjectsInNamespace(apiv1.NamespaceDefault) -} - -func (c *client) DeleteMachineObjectsInNamespace(namespace string) error { +func (c *client) DeleteMachines(namespace string) error { err := c.clientSet.ClusterV1alpha1().Machines(namespace).DeleteCollection(newDeleteOptions(), metav1.ListOptions{}) if err != nil { - return errors.Wrapf(err, "error deleting machine objects in namespace %q", namespace) + return errors.Wrapf(err, "error deleting Machines in namespace %q", namespace) } err = c.waitForMachinesDelete(namespace) if err != nil { - return errors.Wrapf(err, "error waiting for machine(s) deletion to complete in namespace %q", namespace) + return errors.Wrapf(err, "error waiting for Machine(s) deletion to complete in namespace %q", namespace) + } + return nil +} + +func (c *client) ForceDeleteMachine(namespace, name string) error { + machine, err := c.clientSet.ClusterV1alpha1().Machines(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting Machine %s/%s", namespace, name) + } + machine.SetFinalizers([]string{}) + if _, err := c.clientSet.ClusterV1alpha1().Machines(namespace).Update(machine); err != nil { + return errors.Wrapf(err, "error removing finalizer for Machine %s/%s", namespace, name) + } + if err := c.clientSet.ClusterV1alpha1().Machines(namespace).Delete(name, newDeleteOptions()); err != nil { + return errors.Wrapf(err, "error deleting Machine %s/%s", namespace, name) + } + return nil +} + +func (c *client) ForceDeleteMachineSet(namespace, name string) error { + ms, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting MachineSet %s/%s", namespace, name) + } + ms.SetFinalizers([]string{}) + if _, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).Update(ms); err != nil { + return errors.Wrapf(err, "error removing finalizer for MachineSet %s/%s", namespace, name) + } + if err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).Delete(name, newDeleteOptions()); err != nil { + return errors.Wrapf(err, "error deleting MachineSet %s/%s", namespace, name) + } + return nil +} + +func (c *client) ForceDeleteMachineDeployment(namespace, name string) error { + md, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return errors.Wrapf(err, "error getting MachineDeployment %s/%s", namespace, name) + } + md.SetFinalizers([]string{}) + if _, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).Update(md); err != nil { + return errors.Wrapf(err, "error removing finalizer for MachineDeployment %s/%s", namespace, name) + } + if err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).Delete(name, newDeleteOptions()); err != nil { + return errors.Wrapf(err, "error deleting MachineDeployment %s/%s", namespace, name) } return nil } @@ -412,7 +611,7 @@ func newDeleteOptions() *metav1.DeleteOptions { // can be passed as hostname or hostname:port, if port is not present the default port 443 is applied. // TODO: Test this function func (c *client) UpdateClusterObjectEndpoint(clusterEndpoint, clusterName, namespace string) error { - cluster, err := c.GetClusterObject(clusterName, namespace) + cluster, err := c.GetCluster(clusterName, namespace) if err != nil { return err } @@ -442,7 +641,7 @@ func (c *client) WaitForClusterV1alpha1Ready() error { func (c *client) waitForClusterDelete(namespace string) error { return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { - klog.V(2).Infof("Waiting for cluster objects to be deleted...") + klog.V(2).Infof("Waiting for Clusters to be deleted...") response, err := c.clientSet.ClusterV1alpha1().Clusters(namespace).List(metav1.ListOptions{}) if err != nil { return false, nil @@ -454,9 +653,23 @@ func (c *client) waitForClusterDelete(namespace string) error { }) } +func (c *client) waitForMachineClassesDelete(namespace string) error { + return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { + klog.V(2).Infof("Waiting for MachineClasses to be deleted...") + response, err := c.clientSet.ClusterV1alpha1().MachineClasses(namespace).List(metav1.ListOptions{}) + if err != nil { + return false, nil + } + if len(response.Items) > 0 { + return false, nil + } + return true, nil + }) +} + func (c *client) waitForMachineDeploymentsDelete(namespace string) error { return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { - klog.V(2).Infof("Waiting for machine deployment objects to be deleted...") + klog.V(2).Infof("Waiting for MachineDeployments to be deleted...") response, err := c.clientSet.ClusterV1alpha1().MachineDeployments(namespace).List(metav1.ListOptions{}) if err != nil { return false, nil @@ -470,7 +683,7 @@ func (c *client) waitForMachineDeploymentsDelete(namespace string) error { func (c *client) waitForMachineSetsDelete(namespace string) error { return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { - klog.V(2).Infof("Waiting for machine set objects to be deleted...") + klog.V(2).Infof("Waiting for MachineSets to be deleted...") response, err := c.clientSet.ClusterV1alpha1().MachineSets(namespace).List(metav1.ListOptions{}) if err != nil { return false, nil @@ -484,7 +697,7 @@ func (c *client) waitForMachineSetsDelete(namespace string) error { func (c *client) waitForMachinesDelete(namespace string) error { return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { - klog.V(2).Infof("Waiting for machine objects to be deleted...") + klog.V(2).Infof("Waiting for Machines to be deleted...") response, err := c.clientSet.ClusterV1alpha1().Machines(namespace).List(metav1.ListOptions{}) if err != nil { return false, nil @@ -496,6 +709,20 @@ func (c *client) waitForMachinesDelete(namespace string) error { }) } +func (c *client) waitForMachineDelete(namespace, name string) error { + return util.PollImmediate(retryIntervalResourceDelete, timeoutResourceDelete, func() (bool, error) { + klog.V(2).Infof("Waiting for Machine %s/%s to be deleted...", namespace, name) + response, err := c.clientSet.ClusterV1alpha1().Machines(namespace).Get(name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + if response != nil { + return false, nil + } + return true, nil + }) +} + func (c *client) kubectlDelete(manifest string) error { return c.kubectlManifestCmd("delete", manifest) } @@ -627,11 +854,11 @@ func ifErrRemove(pErr *error, path string) { } func GetClusterAPIObject(client Client, clusterName, namespace string) (*clusterv1.Cluster, *clusterv1.Machine, []*clusterv1.Machine, error) { - machines, err := client.GetMachineObjectsInNamespace(namespace) + machines, err := client.GetMachines(namespace) if err != nil { return nil, nil, nil, errors.Wrap(err, "unable to fetch machines") } - cluster, err := client.GetClusterObject(clusterName, namespace) + cluster, err := client.GetCluster(clusterName, namespace) if err != nil { return nil, nil, nil, errors.Wrapf(err, "unable to fetch cluster %s/%s", namespace, clusterName) } diff --git a/cmd/clusterctl/clusterdeployer/clusterdeployer.go b/cmd/clusterctl/clusterdeployer/clusterdeployer.go index 1097650a9e64..9b3334ae8f49 100644 --- a/cmd/clusterctl/clusterdeployer/clusterdeployer.go +++ b/cmd/clusterctl/clusterdeployer/clusterdeployer.go @@ -27,7 +27,6 @@ import ( "sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/provider" "sigs.k8s.io/cluster-api/cmd/clusterctl/phases" clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" - "sigs.k8s.io/cluster-api/pkg/util" ) type ClusterDeployer struct { @@ -118,22 +117,9 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster } } - klog.Infof("Creating namespace %q on target cluster", cluster.Namespace) - addNamespaceToTarget := func() (bool, error) { - err = targetClient.EnsureNamespace(cluster.Namespace) - if err != nil { - return false, nil - } - return true, nil - } - - if err := util.Retry(addNamespaceToTarget, 0); err != nil { - return errors.Wrapf(err, "unable to ensure namespace %q in target cluster", cluster.Namespace) - } - - klog.Info("Applying Cluster API stack to target cluster") - if err := d.applyClusterAPIComponentsWithPivoting(targetClient, bootstrapClient, cluster.Namespace); err != nil { - return errors.Wrap(err, "unable to apply cluster api stack to target cluster") + klog.Info("Pivoting Cluster API stack to target cluster") + if err := phases.Pivot(bootstrapClient, targetClient, d.providerComponents); err != nil { + return errors.Wrap(err, "unable to pivot cluster api stack to target cluster") } klog.Info("Saving provider components to the target cluster") @@ -159,7 +145,7 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster return nil } -func (d *ClusterDeployer) Delete(targetClient clusterclient.Client, namespace string) error { +func (d *ClusterDeployer) Delete(targetClient clusterclient.Client) error { klog.Info("Creating bootstrap cluster") bootstrapClient, cleanupBootstrapCluster, err := phases.CreateBootstrapCluster(d.bootstrapProvisioner, d.cleanupBootstrapCluster, d.clientFactory) defer cleanupBootstrapCluster() @@ -168,24 +154,13 @@ func (d *ClusterDeployer) Delete(targetClient clusterclient.Client, namespace st } defer closeClient(bootstrapClient, "bootstrap") - klog.Info("Applying Cluster API stack to bootstrap cluster") - if err := phases.ApplyClusterAPIComponents(bootstrapClient, d.providerComponents); err != nil { - return errors.Wrap(err, "unable to apply cluster api stack to bootstrap cluster") - } - - klog.Info("Deleting Cluster API Provider Components from target cluster") - if err = targetClient.Delete(d.providerComponents); err != nil { - klog.Infof("error while removing provider components from target cluster: %v", err) - klog.Infof("Continuing with a best effort delete") - } - - klog.Info("Copying objects from target cluster to bootstrap cluster") - if err = pivotNamespace(targetClient, bootstrapClient, namespace); err != nil { - return errors.Wrap(err, "unable to copy objects from target to bootstrap cluster") + klog.Info("Pivoting Cluster API stack to bootstrap cluster") + if err := phases.Pivot(targetClient, bootstrapClient, d.providerComponents); err != nil { + return errors.Wrap(err, "unable to pivot Cluster API stack to bootstrap cluster") } klog.Info("Deleting objects from bootstrap cluster") - if err = deleteObjectsInNamespace(bootstrapClient, namespace); err != nil { + if err := deleteClusterAPIObjectsInAllNamespaces(bootstrapClient); err != nil { return errors.Wrap(err, "unable to finish deleting objects in bootstrap cluster, resources may have been leaked") } @@ -229,112 +204,31 @@ func (d *ClusterDeployer) saveProviderComponentsToCluster(factory provider.Compo return nil } -func (d *ClusterDeployer) applyClusterAPIComponentsWithPivoting(client, source clusterclient.Client, namespace string) error { - klog.Info("Applying Cluster API Provider Components") - if err := client.Apply(d.providerComponents); err != nil { - return errors.Wrap(err, "unable to apply cluster api controllers") - } - - klog.Info("Pivoting Cluster API objects from bootstrap to target cluster.") - err := pivotNamespace(source, client, namespace) - if err != nil { - return errors.Wrap(err, "unable to pivot cluster API objects") - } - - return nil -} - -func pivotNamespace(from, to clusterclient.Client, namespace string) error { - if err := from.WaitForClusterV1alpha1Ready(); err != nil { - return errors.New("cluster v1alpha1 resource not ready on source cluster") - } - - if err := to.WaitForClusterV1alpha1Ready(); err != nil { - return errors.New("cluster v1alpha1 resource not ready on target cluster") - } - - clusters, err := from.GetClusterObjectsInNamespace(namespace) - if err != nil { - return err - } - - for _, cluster := range clusters { - // New objects cannot have a specified resource version. Clear it out. - cluster.SetResourceVersion("") - if err = to.CreateClusterObject(cluster); err != nil { - return errors.Wrapf(err, "error moving Cluster %q", cluster.GetName()) - } - klog.Infof("Moved Cluster '%s'", cluster.GetName()) - } - - fromDeployments, err := from.GetMachineDeploymentObjectsInNamespace(namespace) - if err != nil { - return err - } - for _, deployment := range fromDeployments { - // New objects cannot have a specified resource version. Clear it out. - deployment.SetResourceVersion("") - if err = to.CreateMachineDeploymentObjects([]*clusterv1.MachineDeployment{deployment}, namespace); err != nil { - return errors.Wrapf(err, "error moving MachineDeployment %q", deployment.GetName()) - } - klog.Infof("Moved MachineDeployment %v", deployment.GetName()) - } - - fromMachineSets, err := from.GetMachineSetObjectsInNamespace(namespace) - if err != nil { - return err - } - for _, machineSet := range fromMachineSets { - // New objects cannot have a specified resource version. Clear it out. - machineSet.SetResourceVersion("") - if err := to.CreateMachineSetObjects([]*clusterv1.MachineSet{machineSet}, namespace); err != nil { - return errors.Wrapf(err, "error moving MachineSet %q", machineSet.GetName()) - } - klog.Infof("Moved MachineSet %v", machineSet.GetName()) - } - - machines, err := from.GetMachineObjectsInNamespace(namespace) - if err != nil { - return err - } - - for _, machine := range machines { - // New objects cannot have a specified resource version. Clear it out. - machine.SetResourceVersion("") - machine.SetOwnerReferences(nil) - if err = to.CreateMachineObjects([]*clusterv1.Machine{machine}, namespace); err != nil { - return errors.Wrapf(err, "error moving Machine %q", machine.GetName()) - } - klog.Infof("Moved Machine '%s'", machine.GetName()) - } - return nil -} - -func deleteObjectsInNamespace(client clusterclient.Client, namespace string) error { +func deleteClusterAPIObjectsInAllNamespaces(client clusterclient.Client) error { var errorList []string - klog.Infof("Deleting machine deployments in namespace %q", namespace) - if err := client.DeleteMachineDeploymentObjectsInNamespace(namespace); err != nil { - err = errors.Wrap(err, "error deleting machine deployments") + klog.Infof("Deleting MachineDeployments in all namespaces") + if err := client.DeleteMachineDeployments(""); err != nil { + err = errors.Wrap(err, "error deleting MachineDeployments") errorList = append(errorList, err.Error()) } - klog.Infof("Deleting machine sets in namespace %q", namespace) - if err := client.DeleteMachineSetObjectsInNamespace(namespace); err != nil { - err = errors.Wrap(err, "error deleting machine sets") + klog.Infof("Deleting MachineSets in all namespaces") + if err := client.DeleteMachineSets(""); err != nil { + err = errors.Wrap(err, "error deleting MachineSets") errorList = append(errorList, err.Error()) } - klog.Infof("Deleting machines in namespace %q", namespace) - if err := client.DeleteMachineObjectsInNamespace(namespace); err != nil { - err = errors.Wrap(err, "error deleting machines") + klog.Infof("Deleting Machines in all namespaces") + if err := client.DeleteMachines(""); err != nil { + err = errors.Wrap(err, "error deleting Machines") errorList = append(errorList, err.Error()) } - klog.Infof("Deleting clusters in namespace %q", namespace) - if err := client.DeleteClusterObjectsInNamespace(namespace); err != nil { - err = errors.Wrap(err, "error deleting clusters") + klog.Infof("Deleting MachineClasses in all namespaces") + if err := client.DeleteMachineClasses(""); err != nil { + err = errors.Wrap(err, "error deleting MachineClasses") errorList = append(errorList, err.Error()) } - klog.Infof("Deleting namespace %q", namespace) - if err := client.DeleteNamespace(namespace); err != nil { - err = errors.Wrap(err, "error deleting namespace") + klog.Infof("Deleting Clusters in all namespaces") + if err := client.DeleteClusters(""); err != nil { + err = errors.Wrap(err, "error deleting Clusters") errorList = append(errorList, err.Error()) } if len(errorList) > 0 { diff --git a/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go b/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go index 23362e65d0c5..cabdb58fc8a3 100644 --- a/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go +++ b/cmd/clusterctl/clusterdeployer/clusterdeployer_test.go @@ -89,38 +89,35 @@ func (m *mockProviderComponentsStore) Load() (string, error) { type stringCheckFunc func(string) error type testClusterClient struct { - ApplyErr error - DeleteErr error - WaitForClusterV1alpha1ReadyErr error - GetClusterObjectsErr error - GetClusterObjectErr error - GetClusterObjectsInNamespaceErr error - GetMachineDeploymentObjectsErr error - GetMachineDeploymentObjectsInNamespaceErr error - GetMachineSetObjectsErr error - GetMachineSetObjectsInNamespaceErr error - GetMachineObjectsErr error - GetMachineObjectsInNamespaceErr error - CreateClusterObjectErr error - CreateMachineObjectsErr error - CreateMachineSetObjectsErr error - CreateMachineDeploymentsObjectsErr error - DeleteClusterObjectsErr error - DeleteClusterObjectsInNamespaceErr error - DeleteMachineObjectsErr error - DeleteMachineObjectsInNamespaceErr error - DeleteMachineSetObjectsErr error - DeleteMachineSetObjectsInNamespaceErr error - DeleteMachineDeploymentsObjectsErr error - DeleteMachineDeploymentsObjectsInNamespaceErr error - UpdateClusterObjectEndpointErr error - EnsureNamespaceErr error - DeleteNamespaceErr error - CloseErr error + ApplyErr error + DeleteErr error + WaitForClusterV1alpha1ReadyErr error + GetClustersErr error + GetClusterErr error + GetMachineClassesErr error + GetMachineDeploymentsErr error + GetMachineSetsErr error + GetMachineSetsForMachineDeploymentErr error + GetMachinesForMachineSetErr error + GetMachinesErr error + CreateClusterObjectErr error + CreateMachinesErr error + CreateMachineSetsErr error + CreateMachineDeploymentsErr error + DeleteClustersErr error + DeleteMachineClassesErr error + DeleteMachineDeploymentsErr error + DeleteMachinesErr error + DeleteMachineSetsErr error + UpdateClusterObjectEndpointErr error + EnsureNamespaceErr error + DeleteNamespaceErr error + CloseErr error ApplyFunc stringCheckFunc clusters map[string][]*clusterv1.Cluster + machineClasses map[string][]*clusterv1.MachineClass machineDeployments map[string][]*clusterv1.MachineDeployment machineSets map[string][]*clusterv1.MachineSet machines map[string][]*clusterv1.Machine @@ -150,13 +147,9 @@ func (c *testClusterClient) WaitForClusterV1alpha1Ready() error { return c.WaitForClusterV1alpha1ReadyErr } -func (c *testClusterClient) GetClusterObjects() ([]*clusterv1.Cluster, error) { - return c.clusters[metav1.NamespaceDefault], c.GetClusterObjectsErr -} - -func (c *testClusterClient) GetClusterObject(clusterName, namespace string) (*clusterv1.Cluster, error) { - if c.GetClusterObjectErr != nil { - return nil, c.GetClusterObjectErr +func (c *testClusterClient) GetCluster(clusterName, namespace string) (*clusterv1.Cluster, error) { + if c.GetClusterErr != nil { + return nil, c.GetClusterErr } var cluster *clusterv1.Cluster for _, nc := range c.clusters[namespace] { @@ -168,35 +161,60 @@ func (c *testClusterClient) GetClusterObject(clusterName, namespace string) (*cl return cluster, nil } -func (c *testClusterClient) GetClusterObjectsInNamespace(namespace string) ([]*clusterv1.Cluster, error) { - if c.GetClusterObjectsInNamespaceErr != nil { - return nil, c.GetClusterObjectsInNamespaceErr +func (c *testClusterClient) GetClusters(namespace string) ([]*clusterv1.Cluster, error) { + if c.GetClustersErr != nil { + return nil, c.GetClustersErr } - return c.clusters[namespace], nil -} - -func (c *testClusterClient) GetMachineDeploymentObjects() ([]*clusterv1.MachineDeployment, error) { - return c.machineDeployments[metav1.NamespaceDefault], c.GetMachineDeploymentObjectsErr -} - -func (c *testClusterClient) GetMachineDeploymentObjectsInNamespace(namespace string) ([]*clusterv1.MachineDeployment, error) { - return c.machineDeployments[namespace], c.GetMachineDeploymentObjectsInNamespaceErr -} - -func (c *testClusterClient) GetMachineSetObjects() ([]*clusterv1.MachineSet, error) { - return c.machineSets[metav1.NamespaceDefault], c.GetMachineSetObjectsErr + if namespace != "" { + return c.clusters[namespace], nil + } + var result []*clusterv1.Cluster // nolint + for _, clusters := range c.clusters { + result = append(result, clusters...) + } + return result, nil } -func (c *testClusterClient) GetMachineSetObjectsInNamespace(namespace string) ([]*clusterv1.MachineSet, error) { - return c.machineSets[namespace], c.GetMachineSetObjectsInNamespaceErr +func (c *testClusterClient) GetMachineDeployments(namespace string) ([]*clusterv1.MachineDeployment, error) { + if c.GetMachineDeploymentsErr != nil { + return nil, c.GetMachineDeploymentsErr + } + if namespace != "" { + return c.machineDeployments[namespace], nil + } + var result []*clusterv1.MachineDeployment // nolint + for _, machineDeployments := range c.machineDeployments { + result = append(result, machineDeployments...) + } + return result, nil } -func (c *testClusterClient) GetMachineObjects() ([]*clusterv1.Machine, error) { - return c.machines[metav1.NamespaceDefault], c.GetMachineObjectsErr +func (c *testClusterClient) GetMachineSets(namespace string) ([]*clusterv1.MachineSet, error) { + if c.GetMachineSetsErr != nil { + return nil, c.GetMachineSetsErr + } + if namespace != "" { + return c.machineSets[namespace], nil + } + var result []*clusterv1.MachineSet // nolint + for _, machineSets := range c.machineSets { + result = append(result, machineSets...) + } + return result, nil } -func (c *testClusterClient) GetMachineObjectsInNamespace(namespace string) ([]*clusterv1.Machine, error) { - return c.machines[namespace], c.GetMachineObjectsInNamespaceErr +func (c *testClusterClient) GetMachines(namespace string) ([]*clusterv1.Machine, error) { + if c.GetMachinesErr != nil { + return nil, c.GetMachinesErr + } + if namespace != "" { + return c.machines[namespace], nil + } + var result []*clusterv1.Machine // nolint + for _, machines := range c.machines { + result = append(result, machines...) + } + return result, nil } func (c *testClusterClient) CreateClusterObject(cluster *clusterv1.Cluster) error { @@ -210,81 +228,97 @@ func (c *testClusterClient) CreateClusterObject(cluster *clusterv1.Cluster) erro return c.CreateClusterObjectErr } -func (c *testClusterClient) CreateMachineDeploymentObjects(deployments []*clusterv1.MachineDeployment, namespace string) error { - if c.CreateMachineDeploymentsObjectsErr == nil { +func (c *testClusterClient) CreateMachineDeployments(deployments []*clusterv1.MachineDeployment, namespace string) error { + if c.CreateMachineDeploymentsErr == nil { if c.machineDeployments == nil { c.machineDeployments = make(map[string][]*clusterv1.MachineDeployment) } c.machineDeployments[namespace] = append(c.machineDeployments[namespace], deployments...) return nil } - return c.CreateMachineDeploymentsObjectsErr + return c.CreateMachineDeploymentsErr } -func (c *testClusterClient) CreateMachineSetObjects(machineSets []*clusterv1.MachineSet, namespace string) error { - if c.CreateMachineSetObjectsErr == nil { +func (c *testClusterClient) CreateMachineSets(machineSets []*clusterv1.MachineSet, namespace string) error { + if c.CreateMachineSetsErr == nil { if c.machineSets == nil { c.machineSets = make(map[string][]*clusterv1.MachineSet) } c.machineSets[namespace] = append(c.machineSets[namespace], machineSets...) return nil } - return c.CreateMachineSetObjectsErr + return c.CreateMachineSetsErr } -func (c *testClusterClient) CreateMachineObjects(machines []*clusterv1.Machine, namespace string) error { - if c.CreateMachineObjectsErr == nil { +func (c *testClusterClient) CreateMachines(machines []*clusterv1.Machine, namespace string) error { + if c.CreateMachinesErr == nil { if c.machines == nil { c.machines = make(map[string][]*clusterv1.Machine) } c.machines[namespace] = append(c.machines[namespace], machines...) return nil } - return c.CreateMachineObjectsErr + return c.CreateMachinesErr } -func (c *testClusterClient) DeleteClusterObjectsInNamespace(ns string) error { - if c.DeleteClusterObjectsInNamespaceErr == nil { +func (c *testClusterClient) DeleteClusters(ns string) error { + if c.DeleteClustersErr != nil { + return c.DeleteClustersErr + } + if ns == "" { + c.clusters = make(map[string][]*clusterv1.Cluster) + } else { delete(c.clusters, ns) } - return c.DeleteClusterObjectsInNamespaceErr + return nil } -func (c *testClusterClient) DeleteClusterObjects() error { - return c.DeleteClusterObjectsErr +func (c *testClusterClient) DeleteMachineClasses(ns string) error { + if c.DeleteMachineClassesErr != nil { + return c.DeleteMachineClassesErr + } + if ns == "" { + c.machineClasses = make(map[string][]*clusterv1.MachineClass) + } else { + delete(c.machineClasses, ns) + } + return nil } -func (c *testClusterClient) DeleteMachineDeploymentObjectsInNamespace(ns string) error { - if c.DeleteMachineDeploymentsObjectsInNamespaceErr == nil { +func (c *testClusterClient) DeleteMachineDeployments(ns string) error { + if c.DeleteMachineDeploymentsErr != nil { + return c.DeleteMachineDeploymentsErr + } + if ns == "" { + c.machineDeployments = make(map[string][]*clusterv1.MachineDeployment) + } else { delete(c.machineDeployments, ns) } - return c.DeleteMachineDeploymentsObjectsInNamespaceErr -} - -func (c *testClusterClient) DeleteMachineDeploymentObjects() error { - return c.DeleteMachineDeploymentsObjectsErr + return nil } -func (c *testClusterClient) DeleteMachineSetObjectsInNamespace(ns string) error { - if c.DeleteMachineSetObjectsInNamespaceErr == nil { +func (c *testClusterClient) DeleteMachineSets(ns string) error { + if c.DeleteMachineSetsErr != nil { + return c.DeleteMachineSetsErr + } + if ns == "" { + c.machineSets = make(map[string][]*clusterv1.MachineSet) + } else { delete(c.machineSets, ns) } - return c.DeleteMachineSetObjectsInNamespaceErr -} - -func (c *testClusterClient) DeleteMachineSetObjects() error { - return c.DeleteMachineSetObjectsErr + return nil } -func (c *testClusterClient) DeleteMachineObjectsInNamespace(ns string) error { - if c.DeleteMachineObjectsInNamespaceErr == nil { +func (c *testClusterClient) DeleteMachines(ns string) error { + if c.DeleteMachinesErr != nil { + return c.DeleteMachinesErr + } + if ns == "" { + c.machines = make(map[string][]*clusterv1.Machine) + } else { delete(c.machines, ns) } - return c.DeleteMachineObjectsInNamespaceErr -} - -func (c *testClusterClient) DeleteMachineObjects() error { - return c.DeleteMachineObjectsErr + return nil } func (c *testClusterClient) UpdateClusterObjectEndpoint(string, string, string) error { @@ -295,9 +329,6 @@ func (c *testClusterClient) Close() error { } func (c *testClusterClient) EnsureNamespace(nsName string) error { - if len(c.namespaces) == 0 { - c.namespaces = append(c.namespaces, nsName) - } if exists := contains(c.namespaces, nsName); !exists { c.namespaces = append(c.namespaces, nsName) } @@ -321,6 +352,155 @@ func (c *testClusterClient) DeleteNamespace(namespaceName string) error { return c.DeleteNamespaceErr } +func (c *testClusterClient) ScaleStatefulSet(ns string, name string, scale int32) error { + return nil +} + +func (c *testClusterClient) GetMachineSetsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineSet, error) { + var result []*clusterv1.MachineSet + for _, ms := range c.machineSets[cluster.Namespace] { + if ms.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + result = append(result, ms) + } + } + return result, nil +} + +func (c *testClusterClient) GetMachineDeploymentsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineDeployment, error) { + var result []*clusterv1.MachineDeployment + for _, md := range c.machineDeployments[cluster.Namespace] { + if md.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + result = append(result, md) + } + } + return result, nil +} + +func (c *testClusterClient) GetMachinesForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.Machine, error) { + var result []*clusterv1.Machine + for _, m := range c.machines[cluster.Namespace] { + if m.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + result = append(result, m) + } + } + return result, nil +} + +func (c *testClusterClient) GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error) { + for _, md := range c.machineDeployments[namespace] { + if md.Name == name { + return md, nil + } + } + return nil, nil +} + +func (c *testClusterClient) GetMachineSet(namespace, name string) (*clusterv1.MachineSet, error) { + for _, ms := range c.machineSets[namespace] { + if ms.Name == name { + return ms, nil + } + } + return nil, nil +} + +func (c *testClusterClient) ForceDeleteCluster(namespace, name string) error { + var newClusters []*clusterv1.Cluster + for _, cluster := range c.clusters[namespace] { + if cluster.Name != name { + newClusters = append(newClusters, cluster) + } + } + c.clusters[namespace] = newClusters + return nil +} + +func (c *testClusterClient) ForceDeleteMachine(namespace, name string) error { + var newMachines []*clusterv1.Machine + for _, machine := range c.machines[namespace] { + if machine.Name != name { + newMachines = append(newMachines, machine) + } + } + c.machines[namespace] = newMachines + return nil +} + +func (c *testClusterClient) ForceDeleteMachineSet(namespace, name string) error { + var newMachineSets []*clusterv1.MachineSet + for _, ms := range c.machineSets[namespace] { + if ms.Name != name { + newMachineSets = append(newMachineSets, ms) + } + } + c.machineSets[namespace] = newMachineSets + return nil +} + +func (c *testClusterClient) ForceDeleteMachineDeployment(namespace, name string) error { + var newMachineDeployments []*clusterv1.MachineDeployment + for _, md := range c.machineDeployments[namespace] { + if md.Name != name { + newMachineDeployments = append(newMachineDeployments, md) + } + } + c.machineDeployments[namespace] = newMachineDeployments + return nil +} + +func (c *testClusterClient) GetMachineSetsForMachineDeployment(md *clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) { + if c.GetMachineSetsForMachineDeploymentErr != nil { + return nil, c.GetMachineSetsForMachineDeploymentErr + } + var results []*clusterv1.MachineSet + machineSets, err := c.GetMachineSets(md.Namespace) + if err != nil { + return nil, err + } + for _, ms := range machineSets { + for _, or := range ms.OwnerReferences { + if or.APIVersion == md.APIVersion && or.Kind == md.Kind && or.Name == md.Name { + results = append(results, ms) + } + } + } + return results, nil +} + +func (c *testClusterClient) GetMachinesForMachineSet(ms *clusterv1.MachineSet) ([]*clusterv1.Machine, error) { + if c.GetMachinesForMachineSetErr != nil { + return nil, c.GetMachinesForMachineSetErr + } + var results []*clusterv1.Machine + machines, err := c.GetMachines(ms.Namespace) + if err != nil { + return nil, err + } + for _, m := range machines { + for _, or := range m.OwnerReferences { + if or.APIVersion == ms.APIVersion && or.Kind == ms.Kind && or.Name == ms.Name { + results = append(results, m) + } + } + } + return results, nil +} + +// TODO: implement GetMachineClasses for testClusterClient and add tests +func (c *testClusterClient) GetMachineClasses(namespace string) ([]*clusterv1.MachineClass, error) { + return c.machineClasses[namespace], c.GetMachineClassesErr +} + +// TODO: implement CreateMachineClass for testClusterClient and add tests +func (c *testClusterClient) CreateMachineClass(*clusterv1.MachineClass) error { + return errors.Errorf("CreateMachineClass Not yet implemented.") +} + +// TODO: implement DeleteMachineClass for testClusterClient and add tests +func (c *testClusterClient) DeleteMachineClass(namespace, name string) error { + return errors.Errorf("DeleteMachineClass Not yet implemented.") +} + func contains(s []string, e string) bool { exists := false for _, existingNs := range s { @@ -419,7 +599,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "success no cleaning bootstrap", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -431,7 +611,7 @@ func TestClusterCreate(t *testing.T) { { name: "success create cluster with \"\" namespace and bootstrapClientContext namespace", bootstrapClient: &testClusterClient{contextNamespace: "foo"}, - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, cleanupExternal: true, expectExternalExists: false, expectExternalCreated: true, @@ -444,7 +624,7 @@ func TestClusterCreate(t *testing.T) { { name: "success cluster with \"\" namespace and \"\" bootstrapClientContext namespace", bootstrapClient: &testClusterClient{}, - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, cleanupExternal: true, expectExternalExists: false, expectExternalCreated: true, @@ -456,7 +636,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail ensureNamespace in bootstrap cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{EnsureNamespaceErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{"foo": getClustersForNamespace("foo", 3)}, @@ -478,7 +658,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail provision multiple clusters in a namespace", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{"foo": getClustersForNamespace("foo", 3)}, @@ -489,7 +669,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail provision bootstrap cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -498,7 +678,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail provision bootstrap cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -507,7 +687,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail create clients", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -518,7 +698,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail apply yaml to bootstrap cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{ApplyErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -528,7 +708,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail waiting for api ready on bootstrap cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{WaitForClusterV1alpha1ReadyErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -538,8 +718,8 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail getting bootstrap cluster objects", - targetClient: &testClusterClient{}, - bootstrapClient: &testClusterClient{GetClusterObjectsInNamespaceErr: errors.New("Test failure")}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, + bootstrapClient: &testClusterClient{GetClustersErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, cleanupExternal: true, @@ -548,8 +728,8 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail getting bootstrap machine objects", - targetClient: &testClusterClient{}, - bootstrapClient: &testClusterClient{GetMachineObjectsInNamespaceErr: errors.New("Test failure")}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, + bootstrapClient: &testClusterClient{GetMachinesErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, cleanupExternal: true, @@ -558,7 +738,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail create cluster", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{CreateClusterObjectErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -568,8 +748,8 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail create control plane", - targetClient: &testClusterClient{}, - bootstrapClient: &testClusterClient{CreateMachineObjectsErr: errors.New("Test failure")}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, + bootstrapClient: &testClusterClient{CreateMachinesErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, cleanupExternal: true, @@ -578,7 +758,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail update bootstrap cluster endpoint", - targetClient: &testClusterClient{}, + targetClient: &testClusterClient{ApplyFunc: func(yaml string) error { return nil }}, bootstrapClient: &testClusterClient{UpdateClusterObjectEndpointErr: errors.New("Test failure")}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -618,7 +798,7 @@ func TestClusterCreate(t *testing.T) { }, { name: "fail create nodes", - targetClient: &testClusterClient{CreateMachineObjectsErr: errors.New("Test failure")}, + targetClient: &testClusterClient{CreateMachinesErr: errors.New("Test failure")}, bootstrapClient: &testClusterClient{}, namespaceToExpectedInternalMachines: make(map[string]int), namespaceToInputCluster: map[string][]*clusterv1.Cluster{metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1)}, @@ -674,7 +854,8 @@ func TestClusterCreate(t *testing.T) { pcStore := mockProviderComponentsStore{} pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &pcStore} d := New(p, f, "", "", bootstrapComponent, testcase.cleanupExternal) - inputMachines := generateMachines() + + inputMachines := make(map[string][]*clusterv1.Machine) for namespace, inputClusters := range testcase.namespaceToInputCluster { ns := namespace @@ -685,10 +866,12 @@ func TestClusterCreate(t *testing.T) { var err error for _, inputCluster := range inputClusters { inputCluster.Name = fmt.Sprintf("%s-cluster", ns) - err = d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory) + inputMachines[inputCluster.Name] = generateMachines(inputCluster, ns) + err = d.Create(inputCluster, inputMachines[inputCluster.Name], pd, kubeconfigOut, &pcFactory) if err != nil { break } + testcase.namespaceToExpectedInternalMachines[ns] = testcase.namespaceToExpectedInternalMachines[ns] + len(inputMachines[inputCluster.Name]) } if (testcase.expectErr && err == nil) || (!testcase.expectErr && err != nil) { t.Fatalf("Unexpected error returned. Got: %v, Want Err: %v", err, testcase.expectErr) @@ -699,7 +882,6 @@ func TestClusterCreate(t *testing.T) { if testcase.expectExternalCreated != p.clusterCreated { t.Errorf("Unexpected bootstrap cluster provisioning. Got: %v, Want: %v", p.clusterCreated, testcase.expectExternalCreated) } - testcase.namespaceToExpectedInternalMachines[ns] = len(inputMachines) } if !testcase.expectErr { @@ -722,9 +904,16 @@ func TestClusterCreate(t *testing.T) { t.Fatalf("Unexpected machine count in namespace %q. Got: %d, Want: %d", ns, len(testcase.targetClient.machines[ns]), testcase.namespaceToExpectedInternalMachines[ns]) } - for i := range inputMachines { - if inputMachines[i].Name != testcase.targetClient.machines[ns][i].Name { - t.Fatalf("Unexpected machine name at %v in namespace %q. Got: %v, Want: %v", i, ns, inputMachines[i].Name, testcase.targetClient.machines[ns][i].Name) + clusterMachines := inputMachines[expectedClusterName] + for _, m := range clusterMachines { + found := false + for _, wm := range testcase.targetClient.machines[ns] { + if m.Name == wm.Name { + found = true + } + } + if !found { + t.Fatalf("Unexpected machine name: %s/%s", ns, m.Name) } } @@ -769,12 +958,16 @@ func TestCreateProviderComponentsScenarios(t *testing.T) { f.clusterClients[bootstrapKubeconfig] = &testClusterClient{} f.clusterClients[targetKubeconfig] = &testClusterClient{} - inputCluster := &clusterv1.Cluster{} - inputCluster.Name = "test-cluster" - inputMachines := generateMachines() + inputCluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: metav1.NamespaceDefault, + }, + } + inputMachines := generateMachines(inputCluster, metav1.NamespaceDefault) pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &tc.pcStore} - providerComponentsYaml := "-yaml\ndefinition" - addonsYaml := "-yaml\ndefinition" + providerComponentsYaml := "---\nyaml: definition" + addonsYaml := "---\nyaml: definition" d := New(p, f, providerComponentsYaml, addonsYaml, "", false) err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory) if err == nil && tc.expectedError != "" { @@ -805,28 +998,28 @@ func TestExtractControlPlaneMachine(t *testing.T) { }{ { name: "success_1_control_plane_1_node", - inputMachines: generateMachines(), - expectedControlPlane: generateTestControlPlaneMachine(singleControlPlaneName), - expectedNodes: generateTestNodeMachines([]string{singleNodeName}), + inputMachines: generateMachines(nil, metav1.NamespaceDefault), + expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName), + expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, []string{singleNodeName}), expectedError: nil, }, { name: "success_1_control_plane_multiple_nodes", - inputMachines: generateValidExtractControlPlaneMachineInput([]string{singleControlPlaneName}, multipleNodeNames), - expectedControlPlane: generateTestControlPlaneMachine(singleControlPlaneName), - expectedNodes: generateTestNodeMachines(multipleNodeNames), + inputMachines: generateValidExtractControlPlaneMachineInput(nil, metav1.NamespaceDefault, singleControlPlaneName, multipleNodeNames), + expectedControlPlane: generateTestControlPlaneMachine(nil, metav1.NamespaceDefault, singleControlPlaneName), + expectedNodes: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), expectedError: nil, }, { name: "fail_more_than_1_control_plane_not_allowed", - inputMachines: generateInvalidExtractControlPlaneMachine(multipleControlPlaneNames, multipleNodeNames), + inputMachines: generateInvalidExtractControlPlaneMachine(nil, metav1.NamespaceDefault, multipleControlPlaneNames, multipleNodeNames), expectedControlPlane: nil, expectedNodes: nil, expectedError: errors.New("expected one control plane machine, got: 2"), }, { name: "fail_0_control_plane_not_allowed", - inputMachines: generateTestNodeMachines(multipleNodeNames), + inputMachines: generateTestNodeMachines(nil, metav1.NamespaceDefault, multipleNodeNames), expectedControlPlane: nil, expectedNodes: nil, expectedError: errors.New("expected one control plane machine, got: 0"), @@ -862,17 +1055,44 @@ func TestDeleteCleanupExternalCluster(t *testing.T) { testCases := []struct { name string - namespace string cleanupExternalCluster bool provisionExternalErr error bootstrapClient *testClusterClient targetClient *testClusterClient expectedErrorMessage string }{ - {"success with cleanup", metav1.NamespaceDefault, true, nil, &testClusterClient{}, &testClusterClient{}, ""}, - {"success without cleanup", metav1.NamespaceDefault, false, nil, &testClusterClient{}, &testClusterClient{}, ""}, - {"error with cleanup", metav1.NamespaceDefault, true, nil, &testClusterClient{}, &testClusterClient{GetMachineSetObjectsInNamespaceErr: errors.New("get machine sets error")}, "unable to copy objects from target to bootstrap cluster: get machine sets error"}, - {"error without cleanup", metav1.NamespaceDefault, true, nil, &testClusterClient{}, &testClusterClient{GetMachineSetObjectsInNamespaceErr: errors.New("get machine sets error")}, "unable to copy objects from target to bootstrap cluster: get machine sets error"}, + { + name: "success with cleanup", + cleanupExternalCluster: true, + provisionExternalErr: nil, + bootstrapClient: &testClusterClient{}, + targetClient: &testClusterClient{}, + expectedErrorMessage: "", + }, + { + name: "success without cleanup", + cleanupExternalCluster: false, + provisionExternalErr: nil, + bootstrapClient: &testClusterClient{}, + targetClient: &testClusterClient{}, + expectedErrorMessage: "", + }, + { + name: "error with cleanup", + cleanupExternalCluster: true, + provisionExternalErr: nil, + bootstrapClient: &testClusterClient{}, + targetClient: &testClusterClient{GetMachineSetsErr: errors.New("get machine sets error")}, + expectedErrorMessage: "get machine sets error", + }, + { + name: "error without cleanup", + cleanupExternalCluster: true, + provisionExternalErr: nil, + bootstrapClient: &testClusterClient{}, + targetClient: &testClusterClient{GetMachineSetsErr: errors.New("get machine sets error")}, + expectedErrorMessage: "get machine sets error", + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -883,12 +1103,12 @@ func TestDeleteCleanupExternalCluster(t *testing.T) { f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient f.clusterClients[targetKubeconfig] = tc.targetClient d := New(p, f, "", "", "", tc.cleanupExternalCluster) - err := d.Delete(tc.targetClient, tc.namespace) + err := d.Delete(tc.targetClient) if err != nil || tc.expectedErrorMessage != "" { if err == nil { t.Errorf("expected error %q", tc.expectedErrorMessage) - } else if err.Error() != tc.expectedErrorMessage { - t.Errorf("Unexpected error: got '%v', want: '%v'", err, tc.expectedErrorMessage) + } else if errors.Cause(err).Error() != tc.expectedErrorMessage { + t.Errorf("Unexpected error: got %q, want: %q", errors.Cause(err).Error(), tc.expectedErrorMessage) } } if !tc.cleanupExternalCluster != p.clusterExists { @@ -903,74 +1123,40 @@ func TestClusterDelete(t *testing.T) { const targetKubeconfig = "target" testCases := []struct { - name string - namespace string - provisionExternalErr error - NewCoreClientsetErr error - bootstrapClient *testClusterClient - targetClient *testClusterClient - expectedErrorMessage string - expectedExternalClusterCount int - expectError bool + name string + provisionExternalErr error + NewCoreClientsetErr error + bootstrapClient *testClusterClient + targetClient *testClusterClient + expectedErrorMessage string + expectedExternalClusterCount int + expectedExternalMachineCount int + expectedExternalMachineSetCount int + expectedExternalMachineDeploymentCount int + expectedExternalMachineClassCount int + expectError bool }{ { - name: "success delete 1/1 cluster, 0 clusters remaining", - namespace: metav1.NamespaceDefault, - bootstrapClient: &testClusterClient{ - clusters: map[string][]*clusterv1.Cluster{ - metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1), - }, - machines: map[string][]*clusterv1.Machine{ - metav1.NamespaceDefault: generateMachines(), - }, - machineSets: map[string][]*clusterv1.MachineSet{ - metav1.NamespaceDefault: newMachineSetsFixture(), - }, - machineDeployments: map[string][]*clusterv1.MachineDeployment{ - metav1.NamespaceDefault: newMachineDeploymentsFixture(), - }, - }, + name: "success delete 1/1 cluster", + bootstrapClient: &testClusterClient{}, targetClient: &testClusterClient{ clusters: map[string][]*clusterv1.Cluster{ metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1), }, machines: map[string][]*clusterv1.Machine{ - metav1.NamespaceDefault: generateMachines(), + metav1.NamespaceDefault: generateMachines(nil, metav1.NamespaceDefault), }, machineSets: map[string][]*clusterv1.MachineSet{ - metav1.NamespaceDefault: newMachineSetsFixture(), + metav1.NamespaceDefault: newMachineSetsFixture(metav1.NamespaceDefault), }, machineDeployments: map[string][]*clusterv1.MachineDeployment{ - metav1.NamespaceDefault: newMachineDeploymentsFixture(), + metav1.NamespaceDefault: newMachineDeploymentsFixture(metav1.NamespaceDefault), }, }, - expectedExternalClusterCount: 0, }, { - name: "success delete 1/3 clusters, 2 clusters remaining", - namespace: "foo", - bootstrapClient: &testClusterClient{ - clusters: map[string][]*clusterv1.Cluster{ - "foo": getClustersForNamespace("foo", 1), - "bar": getClustersForNamespace("bar", 1), - "baz": getClustersForNamespace("baz", 1), - }, - machines: map[string][]*clusterv1.Machine{ - "foo": generateMachines(), - "bar": generateMachines(), - "baz": generateMachines(), - }, - machineSets: map[string][]*clusterv1.MachineSet{ - "foo": newMachineSetsFixture(), - "bar": newMachineSetsFixture(), - "baz": newMachineSetsFixture(), - }, - machineDeployments: map[string][]*clusterv1.MachineDeployment{ - "foo": newMachineDeploymentsFixture(), - "bar": newMachineDeploymentsFixture(), - "baz": newMachineDeploymentsFixture(), - }, - }, + name: "success delete 3/3 clusters", + bootstrapClient: &testClusterClient{}, targetClient: &testClusterClient{ clusters: map[string][]*clusterv1.Cluster{ "foo": getClustersForNamespace("foo", 1), @@ -978,53 +1164,48 @@ func TestClusterDelete(t *testing.T) { "baz": getClustersForNamespace("baz", 1), }, machines: map[string][]*clusterv1.Machine{ - "foo": generateMachines(), - "bar": generateMachines(), - "baz": generateMachines(), + "foo": generateMachines(nil, "foo"), + "bar": generateMachines(nil, "bar"), + "baz": generateMachines(nil, "baz"), }, machineSets: map[string][]*clusterv1.MachineSet{ - "foo": newMachineSetsFixture(), - "bar": newMachineSetsFixture(), - "baz": newMachineSetsFixture(), + "foo": newMachineSetsFixture("foo"), + "bar": newMachineSetsFixture("bar"), + "baz": newMachineSetsFixture("baz"), }, machineDeployments: map[string][]*clusterv1.MachineDeployment{ - "foo": newMachineDeploymentsFixture(), - "bar": newMachineDeploymentsFixture(), - "baz": newMachineDeploymentsFixture(), + "foo": newMachineDeploymentsFixture("foo"), + "bar": newMachineDeploymentsFixture("bar"), + "baz": newMachineDeploymentsFixture("baz"), }, }, - expectedExternalClusterCount: 2, }, { name: "error creating core client", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: errors.New("error creating core client"), bootstrapClient: &testClusterClient{}, targetClient: &testClusterClient{}, - expectedErrorMessage: "could not create bootstrap cluster: unable to create bootstrap client: error creating core client", + expectedErrorMessage: "error creating core client", }, { name: "fail provision bootstrap cluster", - namespace: metav1.NamespaceDefault, provisionExternalErr: errors.New("minikube error"), NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, targetClient: &testClusterClient{}, - expectedErrorMessage: "could not create bootstrap cluster: could not create bootstrap control plane: minikube error", + expectedErrorMessage: "minikube error", }, { name: "fail apply yaml to bootstrap cluster", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{ApplyErr: errors.New("yaml apply error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to apply cluster api stack to bootstrap cluster: unable to apply cluster api controllers: yaml apply error", + expectedErrorMessage: "yaml apply error", }, { name: "fail delete provider components should succeed", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, @@ -1033,81 +1214,76 @@ func TestClusterDelete(t *testing.T) { }, { name: "error listing machines", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, - targetClient: &testClusterClient{GetMachineObjectsInNamespaceErr: errors.New("get machines error")}, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: get machines error", + targetClient: &testClusterClient{GetMachinesErr: errors.New("get machines error")}, + expectedErrorMessage: "get machines error", }, { name: "error listing machine sets", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, - targetClient: &testClusterClient{GetMachineSetObjectsInNamespaceErr: errors.New("get machine sets error")}, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: get machine sets error", + targetClient: &testClusterClient{GetMachineSetsErr: errors.New("get machine sets error")}, + expectedErrorMessage: "get machine sets error", }, { name: "error listing machine deployments", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, - targetClient: &testClusterClient{GetMachineDeploymentObjectsInNamespaceErr: errors.New("get machine deployments error")}, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: get machine deployments error", + targetClient: &testClusterClient{GetMachineDeploymentsErr: errors.New("get machine deployments error")}, + expectedErrorMessage: "get machine deployments error", }, { name: "error listing clusters", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{}, - targetClient: &testClusterClient{GetClusterObjectsInNamespaceErr: errors.New("get clusters error")}, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: get clusters error", + targetClient: &testClusterClient{GetClustersErr: errors.New("get clusters error")}, + expectedErrorMessage: "get clusters error", }, { name: "error creating machines", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{CreateMachineObjectsErr: errors.New("create machines error")}, + bootstrapClient: &testClusterClient{CreateMachinesErr: errors.New("create machines error")}, targetClient: &testClusterClient{ machines: map[string][]*clusterv1.Machine{ - metav1.NamespaceDefault: generateMachines(), + metav1.NamespaceDefault: generateMachines(nil, metav1.NamespaceDefault), }, }, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: error moving Machine \"test-control-plane\": create machines error", + expectedErrorMessage: "create machines error", + expectedExternalMachineCount: 2, }, { name: "error creating machine sets", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{CreateMachineSetObjectsErr: errors.New("create machine sets error")}, + bootstrapClient: &testClusterClient{CreateMachineSetsErr: errors.New("create machine sets error")}, targetClient: &testClusterClient{ machineSets: map[string][]*clusterv1.MachineSet{ - metav1.NamespaceDefault: newMachineSetsFixture(), + metav1.NamespaceDefault: newMachineSetsFixture(metav1.NamespaceDefault), }, }, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: error moving MachineSet \"machine-set-name-1\": create machine sets error", + expectedErrorMessage: "create machine sets error", + expectedExternalMachineSetCount: 2, }, { name: "error creating machine deployments", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{CreateMachineDeploymentsObjectsErr: errors.New("create machine deployments error")}, + bootstrapClient: &testClusterClient{CreateMachineDeploymentsErr: errors.New("create machine deployments error")}, targetClient: &testClusterClient{ machineDeployments: map[string][]*clusterv1.MachineDeployment{ - metav1.NamespaceDefault: newMachineDeploymentsFixture(), + metav1.NamespaceDefault: newMachineDeploymentsFixture(metav1.NamespaceDefault), }}, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: error moving MachineDeployment \"machine-deployment-name-1\": create machine deployments error", + expectedErrorMessage: "create machine deployments error", + expectedExternalMachineDeploymentCount: 2, }, { name: "error creating cluster", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, bootstrapClient: &testClusterClient{CreateClusterObjectErr: errors.New("create cluster error")}, @@ -1116,53 +1292,48 @@ func TestClusterDelete(t *testing.T) { metav1.NamespaceDefault: getClustersForNamespace(metav1.NamespaceDefault, 1), }, }, - expectedErrorMessage: "unable to copy objects from target to bootstrap cluster: error moving Cluster \"default-cluster\": create cluster error", + expectedErrorMessage: "create cluster error", + expectedExternalClusterCount: 1, }, - { name: "error deleting machines", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{DeleteMachineObjectsInNamespaceErr: errors.New("delete machines error")}, + bootstrapClient: &testClusterClient{DeleteMachinesErr: errors.New("delete machines error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to finish deleting objects in bootstrap cluster, resources may have been leaked: error(s) encountered deleting objects from bootstrap cluster: [error deleting machines: delete machines error]", + expectedErrorMessage: "error(s) encountered deleting objects from bootstrap cluster: [error deleting Machines: delete machines error]", }, { name: "error deleting machine sets", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{DeleteMachineSetObjectsInNamespaceErr: errors.New("delete machine sets error")}, + bootstrapClient: &testClusterClient{DeleteMachineSetsErr: errors.New("delete machine sets error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to finish deleting objects in bootstrap cluster, resources may have been leaked: error(s) encountered deleting objects from bootstrap cluster: [error deleting machine sets: delete machine sets error]", + expectedErrorMessage: "error(s) encountered deleting objects from bootstrap cluster: [error deleting MachineSets: delete machine sets error]", }, { name: "error deleting machine deployments", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{DeleteMachineDeploymentsObjectsInNamespaceErr: errors.New("delete machine deployments error")}, + bootstrapClient: &testClusterClient{DeleteMachineDeploymentsErr: errors.New("delete machine deployments error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to finish deleting objects in bootstrap cluster, resources may have been leaked: error(s) encountered deleting objects from bootstrap cluster: [error deleting machine deployments: delete machine deployments error]", + expectedErrorMessage: "error(s) encountered deleting objects from bootstrap cluster: [error deleting MachineDeployments: delete machine deployments error]", }, { name: "error deleting clusters", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{DeleteClusterObjectsInNamespaceErr: errors.New("delete clusters error")}, + bootstrapClient: &testClusterClient{DeleteClustersErr: errors.New("delete clusters error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to finish deleting objects in bootstrap cluster, resources may have been leaked: error(s) encountered deleting objects from bootstrap cluster: [error deleting clusters: delete clusters error]", + expectedErrorMessage: "error(s) encountered deleting objects from bootstrap cluster: [error deleting Clusters: delete clusters error]", }, { name: "error deleting machines and clusters", - namespace: metav1.NamespaceDefault, provisionExternalErr: nil, NewCoreClientsetErr: nil, - bootstrapClient: &testClusterClient{DeleteMachineObjectsInNamespaceErr: errors.New("delete machines error"), DeleteClusterObjectsInNamespaceErr: errors.New("delete clusters error")}, + bootstrapClient: &testClusterClient{DeleteMachinesErr: errors.New("delete machines error"), DeleteClustersErr: errors.New("delete clusters error")}, targetClient: &testClusterClient{}, - expectedErrorMessage: "unable to finish deleting objects in bootstrap cluster, resources may have been leaked: error(s) encountered deleting objects from bootstrap cluster: [error deleting machines: delete machines error, error deleting clusters: delete clusters error]", + expectedErrorMessage: "error(s) encountered deleting objects from bootstrap cluster: [error deleting Machines: delete machines error, error deleting Clusters: delete clusters error]", }, } @@ -1170,110 +1341,167 @@ func TestClusterDelete(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { kubeconfigOut := newTempFile(t) defer os.Remove(kubeconfigOut) - p := &testClusterProvisioner{err: testCase.provisionExternalErr, kubeconfig: bootstrapKubeconfig} + p := &testClusterProvisioner{ + err: testCase.provisionExternalErr, + kubeconfig: bootstrapKubeconfig, + } f := newTestClusterClientFactory() f.clusterClients[bootstrapKubeconfig] = testCase.bootstrapClient - f.clusterClients[targetKubeconfig] = testCase.targetClient f.ClusterClientErr = testCase.NewCoreClientsetErr d := New(p, f, "", "", "", true) - err := d.Delete(testCase.targetClient, testCase.namespace) + err := d.Delete(testCase.targetClient) if err != nil || testCase.expectedErrorMessage != "" { if err == nil { t.Errorf("expected error %q", testCase.expectedErrorMessage) - } else if err.Error() != testCase.expectedErrorMessage { - t.Errorf("Unexpected error: got %q, want: %q", err, testCase.expectedErrorMessage) + } else if errors.Cause(err).Error() != testCase.expectedErrorMessage { + t.Errorf("Unexpected error: got %q, want: %q", errors.Cause(err).Error(), testCase.expectedErrorMessage) } } if !testCase.expectError { - if len(testCase.bootstrapClient.clusters[testCase.namespace]) != 0 { - t.Fatalf("Unexpected cluster count in namespace %q. Got: %d, Want: 0", testCase.namespace, len(testCase.targetClient.clusters[testCase.namespace])) + var ( + bootstrapClusters, bootstrapMachines, bootstrapMachineDeployments, bootstrapMachineSets, bootstrapMachineClasses int + targetClusters, targetMachines, targetMachineDeployments, targetMachineSets, targetMachineClasses int + ) + for _, clusters := range testCase.bootstrapClient.clusters { + bootstrapClusters = bootstrapClusters + len(clusters) + } + for _, machines := range testCase.bootstrapClient.machines { + bootstrapMachines = bootstrapMachines + len(machines) + } + for _, machineDeployments := range testCase.bootstrapClient.machineDeployments { + bootstrapMachineDeployments = bootstrapMachineDeployments + len(machineDeployments) + } + for _, machineSets := range testCase.bootstrapClient.machineSets { + bootstrapMachineSets = bootstrapMachineSets + len(machineSets) } - if len(testCase.bootstrapClient.machines[testCase.namespace]) != 0 { - t.Fatalf("Unexpected machine count in namespace %q. Got: %d, Want: 0", testCase.namespace, len(testCase.targetClient.machines[testCase.namespace])) + for _, machineClasses := range testCase.bootstrapClient.machineClasses { + bootstrapMachineClasses = bootstrapMachineClasses + len(machineClasses) } - if len(testCase.bootstrapClient.machineSets[testCase.namespace]) != 0 { - t.Fatalf("Unexpected machineSets count in namespace %q. Got: %d, Want: 0", testCase.namespace, len(testCase.targetClient.machineSets[testCase.namespace])) + for _, clusters := range testCase.targetClient.clusters { + targetClusters = targetClusters + len(clusters) } - if len(testCase.bootstrapClient.machineDeployments[testCase.namespace]) != 0 { - t.Fatalf("Unexpected machineDeployments count in namespace %q. Got: %d, Want: 0", testCase.namespace, len(testCase.targetClient.machineDeployments[testCase.namespace])) + for _, machines := range testCase.targetClient.machines { + targetMachines = targetMachines + len(machines) } - if len(testCase.bootstrapClient.clusters) != testCase.expectedExternalClusterCount { - t.Fatalf("Unexpected remaining cluster count. Got: %d, Want: %d", len(testCase.bootstrapClient.clusters), testCase.expectedExternalClusterCount) + for _, machineDeployments := range testCase.targetClient.machineDeployments { + targetMachineDeployments = targetMachineDeployments + len(machineDeployments) } - if contains(testCase.bootstrapClient.namespaces, testCase.namespace) { - t.Fatalf("Unexpected remaining namespace %q in bootstrap cluster. Got: Found, Want: NotFound", testCase.namespace) + for _, machineSets := range testCase.targetClient.machineSets { + targetMachineSets = targetMachineSets + len(machineSets) } - if testCase.namespace != apiv1.NamespaceDefault && contains(testCase.targetClient.namespaces, testCase.namespace) { - t.Fatalf("Unexpected remaining namespace %q in target cluster. Got: Found, Want: NotFound", testCase.namespace) + for _, machineClasses := range testCase.targetClient.machineClasses { + targetMachineClasses = targetMachineClasses + len(machineClasses) + } + + if bootstrapClusters != 0 { + t.Fatalf("Unexpected Cluster count in bootstrap cluster. Got: %d, Want: 0", bootstrapClusters) + } + if bootstrapMachines != 0 { + t.Fatalf("Unexpected Machine count in bootstrap cluster. Got: %d, Want: 0", bootstrapMachines) + } + if bootstrapMachineSets != 0 { + t.Fatalf("Unexpected MachineSet count in bootstrap cluster. Got: %d, Want: 0", bootstrapMachineSets) + } + if bootstrapMachineDeployments != 0 { + t.Fatalf("Unexpected MachineDeployment count in bootstrap cluster. Got: %d, Want: 0", bootstrapMachineDeployments) + } + if bootstrapMachineClasses != 0 { + t.Fatalf("Unexpected MachineClass count in bootstrap cluster. Got: %d, Want: 0", bootstrapMachineClasses) + } + if targetClusters != testCase.expectedExternalClusterCount { + t.Fatalf("Unexpected Cluster count in target cluster. Got: %d, Want: %d", targetClusters, testCase.expectedExternalClusterCount) + } + if targetMachines != testCase.expectedExternalMachineCount { + t.Fatalf("Unexpected Machine count in target cluster. Got: %d, Want: %d", targetMachines, testCase.expectedExternalMachineCount) + } + if targetMachineSets != testCase.expectedExternalMachineSetCount { + t.Fatalf("Unexpected MachineSet count in target cluster. Got: %d, Want: %d", targetMachineSets, testCase.expectedExternalMachineSetCount) + } + if targetMachineDeployments != testCase.expectedExternalMachineDeploymentCount { + t.Fatalf("Unexpected MachineDeployment count in target cluster. Got: %d, Want: %d", targetMachineDeployments, testCase.expectedExternalMachineDeploymentCount) + } + if targetMachineClasses != testCase.expectedExternalMachineClassCount { + t.Fatalf("Unexpected MachineClass count in target cluster. Got: %d, Want: %d", targetMachineClasses, testCase.expectedExternalMachineClassCount) } } }) } } -func generateTestControlPlaneMachine(name string) *clusterv1.Machine { - return &clusterv1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: clusterv1.MachineSpec{ - Versions: clusterv1.MachineVersionInfo{ - ControlPlane: "1.10.1", - }, +func generateTestControlPlaneMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine { + machine := generateTestNodeMachine(cluster, ns, name) + machine.Spec = clusterv1.MachineSpec{ + Versions: clusterv1.MachineVersionInfo{ + ControlPlane: "1.10.1", }, } + return machine } -func generateTestNodeMachine(name string) *clusterv1.Machine { - return &clusterv1.Machine{ +func generateTestNodeMachine(cluster *clusterv1.Cluster, ns, name string) *clusterv1.Machine { + machine := clusterv1.Machine{ ObjectMeta: metav1.ObjectMeta{ - Name: name, + Name: name, + Namespace: ns, }, } -} - -func generateTestControlPlaneMachines(controlPlaneNames []string) []*clusterv1.Machine { - controlPlanes := make([]*clusterv1.Machine, 0, len(controlPlaneNames)) - for _, mn := range controlPlaneNames { - controlPlanes = append(controlPlanes, generateTestControlPlaneMachine(mn)) + if cluster != nil { + machine.Labels = map[string]string{clusterv1.MachineClusterLabelName: cluster.Name} + machine.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: cluster.APIVersion, + Kind: cluster.Kind, + Name: cluster.Name, + UID: cluster.UID, + }, + } } - return controlPlanes + return &machine } -func generateTestNodeMachines(nodeNames []string) []*clusterv1.Machine { +func generateTestNodeMachines(cluster *clusterv1.Cluster, ns string, nodeNames []string) []*clusterv1.Machine { nodes := make([]*clusterv1.Machine, 0, len(nodeNames)) for _, nn := range nodeNames { - nodes = append(nodes, generateTestNodeMachine(nn)) + nodes = append(nodes, generateTestNodeMachine(cluster, ns, nn)) } return nodes } -func generateInvalidExtractControlPlaneMachine(controlPlaneNames, nodeNames []string) []*clusterv1.Machine { - controlPlanes := generateTestControlPlaneMachines(controlPlaneNames) - nodes := generateTestNodeMachines(nodeNames) - - return append(controlPlanes, nodes...) +func generateInvalidExtractControlPlaneMachine(cluster *clusterv1.Cluster, ns string, controlPlaneNames, nodeNames []string) []*clusterv1.Machine { + var machines []*clusterv1.Machine // nolint + for _, name := range controlPlaneNames { + machines = append(machines, generateTestControlPlaneMachine(cluster, ns, name)) + } + machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...) + return machines } -func generateValidExtractControlPlaneMachineInput(controlPlaneNames, nodeNames []string) []*clusterv1.Machine { - controlPlanes := generateTestControlPlaneMachines(controlPlaneNames) - nodes := generateTestNodeMachines(nodeNames) - - return append(controlPlanes, nodes...) +func generateValidExtractControlPlaneMachineInput(cluster *clusterv1.Cluster, ns, controlPlaneName string, nodeNames []string) []*clusterv1.Machine { + var machines []*clusterv1.Machine + machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName)) + machines = append(machines, generateTestNodeMachines(cluster, ns, nodeNames)...) + return machines } -func generateMachines() []*clusterv1.Machine { - controlPlaneMachine := generateTestControlPlaneMachine("test-control-plane") - node := generateTestNodeMachine("test-node") - return []*clusterv1.Machine{controlPlaneMachine, node} +func generateMachines(cluster *clusterv1.Cluster, ns string) []*clusterv1.Machine { + var machines []*clusterv1.Machine + controlPlaneName := "control-plane" + workerName := "node" + if cluster != nil { + controlPlaneName = cluster.Name + controlPlaneName + workerName = cluster.Name + workerName + } + machines = append(machines, generateTestControlPlaneMachine(cluster, ns, controlPlaneName)) + machines = append(machines, generateTestNodeMachine(cluster, ns, workerName)) + return machines } -func newMachineSetsFixture() []*clusterv1.MachineSet { +func newMachineSetsFixture(ns string) []*clusterv1.MachineSet { return []*clusterv1.MachineSet{ - {ObjectMeta: metav1.ObjectMeta{Name: "machine-set-name-1"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "machine-set-name-2"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine-set-name-1", Namespace: ns}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine-set-name-2", Namespace: ns}}, } } @@ -1290,17 +1518,17 @@ func getClustersForNamespace(namespace string, count int) []*clusterv1.Cluster { return clusters } -func newMachineDeploymentsFixture() []*clusterv1.MachineDeployment { +func newMachineDeploymentsFixture(ns string) []*clusterv1.MachineDeployment { return []*clusterv1.MachineDeployment{ - {ObjectMeta: metav1.ObjectMeta{Name: "machine-deployment-name-1"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "machine-deployment-name-2"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine-deployment-name-1", Namespace: ns}}, + {ObjectMeta: metav1.ObjectMeta{Name: "machine-deployment-name-2", Namespace: ns}}, } } -func newClustersFixture() []*clusterv1.Cluster { +func newClustersFixture(ns string) []*clusterv1.Cluster { return []*clusterv1.Cluster{ - {ObjectMeta: metav1.ObjectMeta{Name: "cluster-name-1"}}, - {ObjectMeta: metav1.ObjectMeta{Name: "cluster-name-2"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "cluster-name-1", Namespace: ns}}, + {ObjectMeta: metav1.ObjectMeta{Name: "cluster-name-2", Namespace: ns}}, } } diff --git a/cmd/clusterctl/cmd/BUILD.bazel b/cmd/clusterctl/cmd/BUILD.bazel index ad5ae4e8a94d..b215680d1257 100644 --- a/cmd/clusterctl/cmd/BUILD.bazel +++ b/cmd/clusterctl/cmd/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "alpha_phase_apply_machines.go", "alpha_phase_create_bootstrap_cluster.go", "alpha_phase_get_kubeconfig.go", + "alpha_phase_pivot.go", "alpha_phases.go", "create.go", "create_cluster.go", diff --git a/cmd/clusterctl/cmd/alpha_phase_pivot.go b/cmd/clusterctl/cmd/alpha_phase_pivot.go new file mode 100644 index 000000000000..9f1984b222dd --- /dev/null +++ b/cmd/clusterctl/cmd/alpha_phase_pivot.go @@ -0,0 +1,100 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "fmt" + "io/ioutil" + + "github.com/spf13/cobra" + "k8s.io/klog" + "sigs.k8s.io/cluster-api/cmd/clusterctl/clusterdeployer/clusterclient" + "sigs.k8s.io/cluster-api/cmd/clusterctl/phases" +) + +type AlphaPhasePivotOptions struct { + SourceKubeconfig string + TargetKubeconfig string + ProviderComponents string +} + +var ppo = &AlphaPhasePivotOptions{} + +var alphaPhasePivotCmd = &cobra.Command{ + Use: "pivot", + Short: "Pivot", + Long: `Pivot`, + Run: func(cmd *cobra.Command, args []string) { + if ppo.ProviderComponents == "" { + exitWithHelp(cmd, "Please provide yaml file for provider components definition.") + } + + if ppo.SourceKubeconfig == "" { + exitWithHelp(cmd, "Please provide a source kubeconfig file.") + } + + if ppo.TargetKubeconfig == "" { + exitWithHelp(cmd, "Please provide a target kubeconfig file.") + } + + if err := RunAlphaPhasePivot(ppo); err != nil { + klog.Exit(err) + } + }, +} + +func RunAlphaPhasePivot(ppo *AlphaPhasePivotOptions) error { + sourceKubeconfig, err := ioutil.ReadFile(ppo.SourceKubeconfig) + if err != nil { + return err + } + + targetKubeconfig, err := ioutil.ReadFile(ppo.TargetKubeconfig) + if err != nil { + return err + } + + providerComponents, err := ioutil.ReadFile(ppo.ProviderComponents) + if err != nil { + return fmt.Errorf("error loading addons file '%v': %v", ppo.ProviderComponents, err) + } + + clientFactory := clusterclient.NewFactory() + sourceClient, err := clientFactory.NewClientFromKubeconfig(string(sourceKubeconfig)) + if err != nil { + return fmt.Errorf("unable to create source cluster client: %v", err) + } + + targetClient, err := clientFactory.NewClientFromKubeconfig(string(targetKubeconfig)) + if err != nil { + return fmt.Errorf("unable to create target cluster client: %v", err) + } + + if err := phases.Pivot(sourceClient, targetClient, string(providerComponents)); err != nil { + return fmt.Errorf("unable to pivot Cluster API Components: %v", err) + } + + return nil +} + +func init() { + // Required flags + alphaPhasePivotCmd.Flags().StringVarP(&ppo.SourceKubeconfig, "source-kubeconfig", "s", "", "Path for the source kubeconfig file to use") + alphaPhasePivotCmd.Flags().StringVarP(&ppo.TargetKubeconfig, "target-kubeconfig", "t", "", "Path for the target kubeconfig file to use") + alphaPhasePivotCmd.Flags().StringVarP(&ppo.ProviderComponents, "provider-components", "p", "", "A yaml file containing provider components to apply to the cluster") + alphaPhasesCmd.AddCommand(alphaPhasePivotCmd) +} diff --git a/cmd/clusterctl/cmd/delete_cluster.go b/cmd/clusterctl/cmd/delete_cluster.go index dd4441c2c6ae..1b683431d679 100644 --- a/cmd/clusterctl/cmd/delete_cluster.go +++ b/cmd/clusterctl/cmd/delete_cluster.go @@ -32,7 +32,6 @@ import ( type DeleteOptions struct { KubeconfigPath string ProviderComponents string - ClusterNamespace string KubeconfigOverrides tcmd.ConfigOverrides BootstrapFlags bootstrap.Options } @@ -61,9 +60,6 @@ func init() { deleteClusterCmd.Flags().StringVarP(&do.KubeconfigPath, "kubeconfig", "", "", "Path to the kubeconfig file to use for connecting to the cluster to be deleted, if empty, the default KUBECONFIG load path is used.") deleteClusterCmd.Flags().StringVarP(&do.ProviderComponents, "provider-components", "p", "", "A yaml file containing cluster api provider controllers and supporting objects, if empty the value is loaded from the cluster's configuration store.") - // Optional flags - deleteClusterCmd.Flags().StringVarP(&do.ClusterNamespace, "cluster-namespace", "", v1.NamespaceDefault, "Namespace where the cluster to be deleted resides") - // BindContextFlags will bind the flags cluster, namespace, and user tcmd.BindContextFlags(&do.KubeconfigOverrides.Context, deleteClusterCmd.Flags(), tcmd.RecommendedContextOverrideFlags("")) @@ -95,7 +91,7 @@ func RunDelete() error { "", do.BootstrapFlags.Cleanup) - return deployer.Delete(clusterClient, do.ClusterNamespace) + return deployer.Delete(clusterClient) } func loadProviderComponents() (string, error) { diff --git a/cmd/clusterctl/phases/BUILD.bazel b/cmd/clusterctl/phases/BUILD.bazel index 188254daa4bc..716245bf44d9 100644 --- a/cmd/clusterctl/phases/BUILD.bazel +++ b/cmd/clusterctl/phases/BUILD.bazel @@ -1,4 +1,4 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", @@ -10,6 +10,7 @@ go_library( "applymachines.go", "createbootstrapcluster.go", "getkubeconfig.go", + "pivot.go", ], importpath = "sigs.k8s.io/cluster-api/cmd/clusterctl/phases", visibility = ["//visibility:public"], @@ -20,6 +21,18 @@ go_library( "//pkg/apis/cluster/v1alpha1:go_default_library", "//pkg/util:go_default_library", "//vendor/github.com/pkg/errors:go_default_library", + "//vendor/k8s.io/api/apps/v1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/util/yaml:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) + +go_test( + name = "go_default_test", + srcs = ["pivot_test.go"], + embed = [":go_default_library"], + deps = [ + "//pkg/apis/cluster/v1alpha1:go_default_library", + "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", + ], +) diff --git a/cmd/clusterctl/phases/applymachines.go b/cmd/clusterctl/phases/applymachines.go index 2c11ccc0276a..ffc8f1af1425 100644 --- a/cmd/clusterctl/phases/applymachines.go +++ b/cmd/clusterctl/phases/applymachines.go @@ -34,7 +34,7 @@ func ApplyMachines(client clusterclient.Client, namespace string, machines []*cl } klog.Infof("Creating machines in namespace %q", namespace) - if err := client.CreateMachineObjects(machines, namespace); err != nil { + if err := client.CreateMachines(machines, namespace); err != nil { return err } diff --git a/cmd/clusterctl/phases/pivot.go b/cmd/clusterctl/phases/pivot.go new file mode 100644 index 000000000000..5ed4105760e8 --- /dev/null +++ b/cmd/clusterctl/phases/pivot.go @@ -0,0 +1,424 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package phases + +import ( + "io" + "strings" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/klog" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +type sourceClient interface { + Delete(string) error + DeleteMachineClass(namespace, name string) error + ForceDeleteCluster(string, string) error + ForceDeleteMachine(string, string) error + ForceDeleteMachineDeployment(string, string) error + ForceDeleteMachineSet(namespace, name string) error + GetClusters(string) ([]*clusterv1.Cluster, error) + GetMachineClasses(string) ([]*clusterv1.MachineClass, error) + GetMachineDeployments(string) ([]*clusterv1.MachineDeployment, error) + GetMachineDeploymentsForCluster(*clusterv1.Cluster) ([]*clusterv1.MachineDeployment, error) + GetMachines(namespace string) ([]*clusterv1.Machine, error) + GetMachineSets(namespace string) ([]*clusterv1.MachineSet, error) + GetMachineSetsForCluster(*clusterv1.Cluster) ([]*clusterv1.MachineSet, error) + GetMachineSetsForMachineDeployment(*clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) + GetMachinesForCluster(*clusterv1.Cluster) ([]*clusterv1.Machine, error) + GetMachinesForMachineSet(*clusterv1.MachineSet) ([]*clusterv1.Machine, error) + ScaleStatefulSet(string, string, int32) error + WaitForClusterV1alpha1Ready() error +} + +type targetClient interface { + Apply(string) error + CreateClusterObject(*clusterv1.Cluster) error + CreateMachineClass(*clusterv1.MachineClass) error + CreateMachineDeployments([]*clusterv1.MachineDeployment, string) error + CreateMachines([]*clusterv1.Machine, string) error + CreateMachineSets([]*clusterv1.MachineSet, string) error + EnsureNamespace(string) error + GetMachineDeployment(namespace, name string) (*clusterv1.MachineDeployment, error) + GetMachineSet(string, string) (*clusterv1.MachineSet, error) + WaitForClusterV1alpha1Ready() error +} + +// Pivot deploys the provided provider components to a target cluster and then migrates +// all cluster-api resources from the source cluster to the target cluster +func Pivot(source sourceClient, target targetClient, providerComponents string) error { + klog.Info("Applying Cluster API Provider Components to Target Cluster") + if err := target.Apply(providerComponents); err != nil { + return errors.Wrap(err, "unable to apply cluster api controllers") + } + + klog.Info("Pivoting Cluster API objects from bootstrap to target cluster.") + if err := pivot(source, target, providerComponents); err != nil { + return errors.Wrap(err, "unable to pivot cluster API objects") + } + + return nil +} + +func pivot(from sourceClient, to targetClient, providerComponents string) error { + // TODO: Attempt to handle rollback in case of pivot failure + + klog.V(4).Info("Ensuring cluster v1alpha1 resources are available on the source cluster") + if err := from.WaitForClusterV1alpha1Ready(); err != nil { + return errors.New("cluster v1alpha1 resource not ready on source cluster") + } + + klog.V(4).Info("Ensuring cluster v1alpha1 resources are available on the target cluster") + if err := to.WaitForClusterV1alpha1Ready(); err != nil { + return errors.New("cluster v1alpha1 resource not ready on target cluster") + } + + klog.V(4).Info("Parsing list of cluster-api controllers from provider components") + controllers, err := parseControllers(providerComponents) + if err != nil { + return errors.Wrap(err, "Failed to extract Cluster API Controllers from the provider components") + } + + // Scale down the controller managers in the source cluster + for _, controller := range controllers { + klog.V(4).Infof("Scaling down controller %s/%s", controller.Namespace, controller.Name) + if err := from.ScaleStatefulSet(controller.Namespace, controller.Name, 0); err != nil { + return errors.Wrapf(err, "Failed to scale down %s/%s", controller.Namespace, controller.Name) + } + } + + klog.V(4).Info("Retrieving list of MachineClasses to move") + machineClasses, err := from.GetMachineClasses("") + if err != nil { + return err + } + + if err := copyMachineClasses(from, to, machineClasses); err != nil { + return err + } + + klog.V(4).Info("Retrieving list of Clusters to move") + clusters, err := from.GetClusters("") + if err != nil { + return err + } + + if err := moveClusters(from, to, clusters); err != nil { + return err + } + + klog.V(4).Info("Retrieving list of MachineDeployments not associated with a Cluster to move") + machineDeployments, err := from.GetMachineDeployments("") + if err != nil { + return err + } + if err := moveMachineDeployments(from, to, machineDeployments); err != nil { + return err + } + + klog.V(4).Info("Retrieving list of MachineSets not associated with a MachineDeployment or a Cluster to move") + machineSets, err := from.GetMachineSets("") + if err != nil { + return err + } + if err := moveMachineSets(from, to, machineSets); err != nil { + return err + } + + klog.V(4).Infof("Retrieving list of Machines not associated with a MachineSet or a Cluster to move") + machines, err := from.GetMachines("") + if err != nil { + return err + } + if err := moveMachines(from, to, machines); err != nil { + return err + } + + if err := deleteMachineClasses(from, machineClasses); err != nil { + return err + } + + klog.V(4).Infof("Deleting provider components from source cluster") + if err := from.Delete(providerComponents); err != nil { + klog.Warningf("Could not delete the provider components from the source cluster: %v", err) + } + + return nil +} + +func moveClusters(from sourceClient, to targetClient, clusters []*clusterv1.Cluster) error { + clusterNames := make([]string, 0, len(clusters)) + for _, c := range clusters { + clusterNames = append(clusterNames, c.Name) + } + klog.V(4).Infof("Preparing to move Clusters: %v", clusterNames) + + for _, c := range clusters { + if err := moveCluster(from, to, c); err != nil { + return errors.Wrapf(err, "Failed to move cluster: %s/%s", c.Namespace, c.Name) + } + } + return nil +} + +func deleteMachineClasses(client sourceClient, machineClasses []*clusterv1.MachineClass) error { + machineClassNames := make([]string, 0, len(machineClasses)) + for _, mc := range machineClasses { + machineClassNames = append(machineClassNames, mc.Name) + } + klog.V(4).Infof("Preparing to delete MachineClasses: %v", machineClassNames) + + for _, mc := range machineClasses { + if err := deleteMachineClass(client, mc); err != nil { + return errors.Wrapf(err, "failed to delete MachineClass %s:%s", mc.Namespace, mc.Name) + } + } + return nil +} + +func deleteMachineClass(client sourceClient, machineClass *clusterv1.MachineClass) error { + // New objects cannot have a specified resource version. Clear it out. + machineClass.SetResourceVersion("") + if err := client.DeleteMachineClass(machineClass.Namespace, machineClass.Name); err != nil { + return errors.Wrapf(err, "error deleting MachineClass %s/%s from source cluster", machineClass.Namespace, machineClass.Name) + } + + klog.V(4).Infof("Successfully deleted MachineClass %s/%s from source cluster", machineClass.Namespace, machineClass.Name) + return nil +} + +func copyMachineClasses(from sourceClient, to targetClient, machineClasses []*clusterv1.MachineClass) error { + machineClassNames := make([]string, 0, len(machineClasses)) + for _, mc := range machineClasses { + machineClassNames = append(machineClassNames, mc.Name) + } + klog.V(4).Infof("Preparing to copy MachineClasses: %v", machineClassNames) + + for _, mc := range machineClasses { + if err := copyMachineClass(from, to, mc); err != nil { + return errors.Wrapf(err, "failed to copy MachineClass %s:%s", mc.Namespace, mc.Name) + } + } + return nil +} + +func copyMachineClass(from sourceClient, to targetClient, machineClass *clusterv1.MachineClass) error { + // New objects cannot have a specified resource version. Clear it out. + machineClass.SetResourceVersion("") + if err := to.CreateMachineClass(machineClass); err != nil { + return errors.Wrapf(err, "error copying MachineClass %s/%s to target cluster", machineClass.Namespace, machineClass.Name) + } + + klog.V(4).Infof("Successfully copied MachineClass %s/%s", machineClass.Namespace, machineClass.Name) + return nil +} + +func moveCluster(from sourceClient, to targetClient, cluster *clusterv1.Cluster) error { + klog.V(4).Infof("Moving Cluster %s/%s", cluster.Namespace, cluster.Name) + + klog.V(4).Infof("Ensuring namespace %q exists on target cluster", cluster.Namespace) + if err := to.EnsureNamespace(cluster.Namespace); err != nil { + return errors.Wrapf(err, "unable to ensure namespace %q in target cluster", cluster.Namespace) + } + + // New objects cannot have a specified resource version. Clear it out. + cluster.SetResourceVersion("") + if err := to.CreateClusterObject(cluster); err != nil { + return errors.Wrapf(err, "error copying Cluster %s/%s to target cluster", cluster.Namespace, cluster.Name) + } + + klog.V(4).Infof("Retrieving list of MachineDeployments to move for Cluster %s/%s", cluster.Namespace, cluster.Name) + machineDeployments, err := from.GetMachineDeploymentsForCluster(cluster) + if err != nil { + return err + } + if err := moveMachineDeployments(from, to, machineDeployments); err != nil { + return err + } + + klog.V(4).Infof("Retrieving list of MachineSets not associated with a MachineDeployment to move for Cluster %s/%s", cluster.Namespace, cluster.Name) + machineSets, err := from.GetMachineSetsForCluster(cluster) + if err != nil { + return err + } + if err := moveMachineSets(from, to, machineSets); err != nil { + return err + } + + klog.V(4).Infof("Retrieving list of Machines not associated with a MachineSet to move for Cluster %s/%s", cluster.Namespace, cluster.Name) + machines, err := from.GetMachinesForCluster(cluster) + if err != nil { + return err + } + if err := moveMachines(from, to, machines); err != nil { + return err + } + + if err := from.ForceDeleteCluster(cluster.Namespace, cluster.Name); err != nil { + return errors.Wrapf(err, "error force deleting cluster %s/%s", cluster.Namespace, cluster.Name) + } + + klog.V(4).Infof("Successfully moved Cluster %s/%s", cluster.Namespace, cluster.Name) + return nil +} + +func moveMachineDeployments(from sourceClient, to targetClient, machineDeployments []*clusterv1.MachineDeployment) error { + machineDeploymentNames := make([]string, 0, len(machineDeployments)) + for _, md := range machineDeployments { + machineDeploymentNames = append(machineDeploymentNames, md.Name) + } + klog.V(4).Infof("Preparing to move MachineDeployments: %v", machineDeploymentNames) + + for _, md := range machineDeployments { + if err := moveMachineDeployment(from, to, md); err != nil { + return errors.Wrapf(err, "failed to move MachineDeployment %s:%s", md.Namespace, md.Name) + } + } + return nil +} + +func moveMachineDeployment(from sourceClient, to targetClient, md *clusterv1.MachineDeployment) error { + klog.V(4).Infof("Moving MachineDeployment %s/%s", md.Namespace, md.Name) + klog.V(4).Infof("Retrieving list of MachineSets for MachineDeployment %s/%s", md.Namespace, md.Name) + machineSets, err := from.GetMachineSetsForMachineDeployment(md) + if err != nil { + return err + } + + if err := moveMachineSets(from, to, machineSets); err != nil { + return err + } + + // New objects cannot have a specified resource version. Clear it out. + md.SetResourceVersion("") + + // Remove owner reference. This currently assumes that the only owner reference would be a Cluster. + md.SetOwnerReferences(nil) + + if err := to.CreateMachineDeployments([]*clusterv1.MachineDeployment{md}, md.Namespace); err != nil { + return errors.Wrapf(err, "error copying MachineDeployment %s/%s to target cluster", md.Namespace, md.Name) + } + + if err := from.ForceDeleteMachineDeployment(md.Namespace, md.Name); err != nil { + return errors.Wrapf(err, "error force deleting MachineDeployment %s/%s from source cluster", md.Namespace, md.Name) + } + klog.V(4).Infof("Successfully moved MachineDeployment %s/%s", md.Namespace, md.Name) + return nil +} + +func moveMachineSets(from sourceClient, to targetClient, machineSets []*clusterv1.MachineSet) error { + machineSetNames := make([]string, 0, len(machineSets)) + for _, ms := range machineSets { + machineSetNames = append(machineSetNames, ms.Name) + } + klog.V(4).Infof("Preparing to move MachineSets: %v", machineSetNames) + + for _, ms := range machineSets { + if err := moveMachineSet(from, to, ms); err != nil { + return errors.Wrapf(err, "failed to move MachineSet %s:%s", ms.Namespace, ms.Name) + } + } + return nil +} + +func moveMachineSet(from sourceClient, to targetClient, ms *clusterv1.MachineSet) error { + klog.V(4).Infof("Moving MachineSet %s/%s", ms.Namespace, ms.Name) + klog.V(4).Infof("Retrieving list of Machines for MachineSet %s/%s", ms.Namespace, ms.Name) + machines, err := from.GetMachinesForMachineSet(ms) + if err != nil { + return err + } + + if err := moveMachines(from, to, machines); err != nil { + return err + } + + // New objects cannot have a specified resource version. Clear it out. + ms.SetResourceVersion("") + + // Remove owner reference. This currently assumes that the only owner references would be a MachineDeployment and/or a Cluster. + ms.SetOwnerReferences(nil) + + if err := to.CreateMachineSets([]*clusterv1.MachineSet{ms}, ms.Namespace); err != nil { + return errors.Wrapf(err, "error copying MachineSet %s/%s to target cluster", ms.Namespace, ms.Name) + } + if err := from.ForceDeleteMachineSet(ms.Namespace, ms.Name); err != nil { + return errors.Wrapf(err, "error force deleting MachineSet %s/%s from source cluster", ms.Namespace, ms.Name) + } + klog.V(4).Infof("Successfully moved MachineSet %s/%s", ms.Namespace, ms.Name) + return nil +} + +func moveMachines(from sourceClient, to targetClient, machines []*clusterv1.Machine) error { + machineNames := make([]string, 0, len(machines)) + for _, m := range machines { + machineNames = append(machineNames, m.Name) + } + klog.V(4).Infof("Preparing to move Machines: %v", machineNames) + + for _, m := range machines { + if err := moveMachine(from, to, m); err != nil { + return errors.Wrapf(err, "failed to move Machine %s:%s", m.Namespace, m.Name) + } + } + return nil +} + +func moveMachine(from sourceClient, to targetClient, m *clusterv1.Machine) error { + klog.V(4).Infof("Moving Machine %s/%s", m.Namespace, m.Name) + + // New objects cannot have a specified resource version. Clear it out. + m.SetResourceVersion("") + + // Remove owner reference. This currently assumes that the only owner references would be a MachineSet and/or a Cluster. + m.SetOwnerReferences(nil) + + if err := to.CreateMachines([]*clusterv1.Machine{m}, m.Namespace); err != nil { + return errors.Wrapf(err, "error copying Machine %s/%s to target cluster", m.Namespace, m.Name) + } + if err := from.ForceDeleteMachine(m.Namespace, m.Name); err != nil { + return errors.Wrapf(err, "error force deleting Machine %s/%s from source cluster", m.Namespace, m.Name) + } + klog.V(4).Infof("Successfully moved Machine %s/%s", m.Namespace, m.Name) + return nil +} + +func parseControllers(providerComponents string) ([]*appsv1.StatefulSet, error) { + decoder := yaml.NewYAMLOrJSONDecoder(strings.NewReader(providerComponents), 32) + + controllers := []*appsv1.StatefulSet{} + + for { + var out appsv1.StatefulSet + err := decoder.Decode(&out) + + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + if out.TypeMeta.Kind == "StatefulSet" { + controllers = append(controllers, &out) + } + } + + return controllers, nil +} diff --git a/cmd/clusterctl/phases/pivot_test.go b/cmd/clusterctl/phases/pivot_test.go new file mode 100644 index 000000000000..2c7dc5a1ce14 --- /dev/null +++ b/cmd/clusterctl/phases/pivot_test.go @@ -0,0 +1,543 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package phases + +import ( + "errors" + "fmt" + "strings" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" +) + +// sourcer map keys are namespaces +// This is intentionally a different implementation than clusterclient +type sourcer struct { + clusters map[string][]*clusterv1.Cluster + machineDeployments map[string][]*clusterv1.MachineDeployment + machineSets map[string][]*clusterv1.MachineSet + machines map[string][]*clusterv1.Machine + machineClasses map[string][]*clusterv1.MachineClass +} + +func newSourcer() *sourcer { + return &sourcer{ + clusters: make(map[string][]*clusterv1.Cluster), + machineDeployments: make(map[string][]*clusterv1.MachineDeployment), + machineSets: make(map[string][]*clusterv1.MachineSet), + machines: make(map[string][]*clusterv1.Machine), + machineClasses: make(map[string][]*clusterv1.MachineClass), + } +} +func (s *sourcer) WithCluster(ns, name string) *sourcer { + s.clusters[ns] = append(s.clusters[ns], &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }) + return s +} + +func (s *sourcer) WithMachineDeployment(ns, cluster, name string) *sourcer { + md := clusterv1.MachineDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } + + if cluster != "" { + md.Labels = map[string]string{clusterv1.MachineClusterLabelName: cluster} + blockOwnerDeletion := true + md.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: clusterv1.SchemeGroupVersion.Version, + Kind: "Cluster", + Name: cluster, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + } + } + s.machineDeployments[ns] = append(s.machineDeployments[ns], &md) + return s +} + +func (s *sourcer) WithMachineSet(ns, cluster, md, name string) *sourcer { + ms := clusterv1.MachineSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } + if cluster != "" { + ms.Labels = map[string]string{clusterv1.MachineClusterLabelName: cluster} + blockOwnerDeletion := true + ms.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: clusterv1.SchemeGroupVersion.Version, + Kind: "Cluster", + Name: cluster, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + } + } + if md != "" { + isController := true + ms.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: clusterv1.SchemeGroupVersion.Version, + Kind: "MachineDeployment", + Name: md, + Controller: &isController, + }, + } + } + + s.machineSets[ns] = append(s.machineSets[ns], &ms) + return s +} + +func (s *sourcer) WithMachine(ns, cluster, ms, name string) *sourcer { + m := clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + } + if cluster != "" { + m.Labels = map[string]string{clusterv1.MachineClusterLabelName: cluster} + blockOwnerDeletion := true + m.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: clusterv1.SchemeGroupVersion.Version, + Kind: "Cluster", + Name: cluster, + BlockOwnerDeletion: &blockOwnerDeletion, + }, + } + } + if ms != "" { + isController := true + m.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: clusterv1.SchemeGroupVersion.Version, + Kind: "MachineSet", + Name: ms, + Controller: &isController, + }, + } + } + + s.machines[ns] = append(s.machines[ns], &m) + return s +} + +func (s *sourcer) WithMachineClass(ns, name string) *sourcer { + s.machineClasses[ns] = append(s.machineClasses[ns], &clusterv1.MachineClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + }) + return s +} + +// Interface implementation below + +func (s *sourcer) Delete(string) error { + return nil +} +func (s *sourcer) DeleteMachineClass(ns, name string) error { + newMachineClasses := []*clusterv1.MachineClass{} + for _, mc := range s.machineClasses[ns] { + if mc.Name != name { + newMachineClasses = append(newMachineClasses, mc) + } + } + s.machineClasses[ns] = newMachineClasses + return nil +} +func (s *sourcer) ForceDeleteCluster(ns, name string) error { + newClusters := []*clusterv1.Cluster{} + for _, d := range s.clusters[ns] { + if d.Name != name { + newClusters = append(newClusters, d) + } + } + s.clusters[ns] = newClusters + return nil +} +func (s *sourcer) ForceDeleteMachine(ns, name string) error { + newMachines := []*clusterv1.Machine{} + for _, d := range s.machines[ns] { + if d.Name != name { + newMachines = append(newMachines, d) + } + } + s.machines[ns] = newMachines + return nil +} +func (s *sourcer) ForceDeleteMachineDeployment(ns, name string) error { + newDeployments := []*clusterv1.MachineDeployment{} + for _, d := range s.machineDeployments[ns] { + if d.Name != name { + newDeployments = append(newDeployments, d) + } + } + s.machineDeployments[ns] = newDeployments + return nil +} +func (s *sourcer) ForceDeleteMachineSet(ns, name string) error { + newSets := []*clusterv1.MachineSet{} + for _, d := range s.machineSets[ns] { + if d.Name != name { + newSets = append(newSets, d) + } + } + s.machineSets[ns] = newSets + return nil +} +func (s *sourcer) GetClusters(ns string) ([]*clusterv1.Cluster, error) { + // empty ns implies all namespaces + if ns == "" { + out := []*clusterv1.Cluster{} + for _, clusters := range s.clusters { + for _, cluster := range clusters { + out = append(out, cluster) + } + } + return out, nil + } + return s.clusters[ns], nil +} +func (s *sourcer) GetMachineClasses(ns string) ([]*clusterv1.MachineClass, error) { + // empty ns implies all namespaces + if ns == "" { + out := []*clusterv1.MachineClass{} + for _, mcs := range s.machineClasses { + for _, mc := range mcs { + out = append(out, mc) + } + } + return out, nil + } + return s.machineClasses[ns], nil +} +func (s *sourcer) GetMachineDeployments(ns string) ([]*clusterv1.MachineDeployment, error) { + // empty ns implies all namespaces + if ns == "" { + out := []*clusterv1.MachineDeployment{} + for _, mds := range s.machineDeployments { + for _, md := range mds { + out = append(out, md) + } + } + return out, nil + } + return s.machineDeployments[ns], nil +} +func (s *sourcer) GetMachineDeploymentsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineDeployment, error) { + var mds []*clusterv1.MachineDeployment + for _, md := range s.machineDeployments[cluster.Namespace] { + if md.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + mds = append(mds, md) + } + } + return mds, nil +} +func (s *sourcer) GetMachines(ns string) ([]*clusterv1.Machine, error) { + // empty ns implies all namespaces + if ns == "" { + out := []*clusterv1.Machine{} + for _, machines := range s.machines { + for _, m := range machines { + out = append(out, m) + } + } + return out, nil + } + return s.machines[ns], nil +} + +func (s *sourcer) GetMachineSets(ns string) ([]*clusterv1.MachineSet, error) { + // empty ns implies all namespaces + if ns == "" { + out := []*clusterv1.MachineSet{} + for _, machineSets := range s.machineSets { + for _, ms := range machineSets { + out = append(out, ms) + } + } + return out, nil + } + return s.machineSets[ns], nil +} + +func (s *sourcer) GetMachineSetsForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.MachineSet, error) { + var machineSets []*clusterv1.MachineSet + for _, ms := range s.machineSets[cluster.Namespace] { + if ms.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + machineSets = append(machineSets, ms) + } + } + return machineSets, nil +} + +func (s *sourcer) GetMachinesForCluster(cluster *clusterv1.Cluster) ([]*clusterv1.Machine, error) { + var machines []*clusterv1.Machine + for _, m := range s.machines[cluster.Namespace] { + if m.Labels[clusterv1.MachineClusterLabelName] == cluster.Name { + machines = append(machines, m) + } + } + return machines, nil +} +func (s *sourcer) GetMachineSetsForMachineDeployment(d *clusterv1.MachineDeployment) ([]*clusterv1.MachineSet, error) { + var machineSets []*clusterv1.MachineSet + for _, ms := range s.machineSets[d.Namespace] { + for _, or := range ms.OwnerReferences { + if or.Kind == "MachineDeployment" && or.Name == d.Name { + machineSets = append(machineSets, ms) + } + } + } + return machineSets, nil +} + +func (s *sourcer) GetMachinesForMachineSet(ms *clusterv1.MachineSet) ([]*clusterv1.Machine, error) { + var machines []*clusterv1.Machine + for _, m := range s.machines[ms.Namespace] { + for _, or := range m.OwnerReferences { + if or.Kind == "MachineSet" && or.Name == ms.Name { + machines = append(machines, m) + } + } + } + return machines, nil +} + +func (s *sourcer) ScaleStatefulSet(string, string, int32) error { + return nil +} +func (s *sourcer) WaitForClusterV1alpha1Ready() error { + return nil +} + +type target struct { + clusters map[string][]*clusterv1.Cluster + machineDeployments map[string][]*clusterv1.MachineDeployment + machineSets map[string][]*clusterv1.MachineSet + machines map[string][]*clusterv1.Machine + machineClasses map[string][]*clusterv1.MachineClass +} + +func (t *target) Apply(string) error { + return nil +} +func (t *target) CreateClusterObject(c *clusterv1.Cluster) error { + t.clusters[c.Namespace] = append(t.clusters[c.Namespace], c) + return nil +} +func (t *target) CreateMachineClass(mc *clusterv1.MachineClass) error { + t.machineClasses[mc.Namespace] = append(t.machineClasses[mc.Namespace], mc) + return nil +} +func (t *target) CreateMachineDeployments(deployments []*clusterv1.MachineDeployment, ns string) error { + t.machineDeployments[ns] = append(t.machineDeployments[ns], deployments...) + return nil +} +func (t *target) CreateMachines(machines []*clusterv1.Machine, ns string) error { + t.machines[ns] = append(t.machines[ns], machines...) + return nil +} +func (t *target) CreateMachineSets(ms []*clusterv1.MachineSet, ns string) error { + t.machineSets[ns] = append(t.machineSets[ns], ms...) + return nil +} + +func (t *target) EnsureNamespace(string) error { + return nil +} +func (t *target) GetMachineDeployment(ns, name string) (*clusterv1.MachineDeployment, error) { + for _, deployment := range t.machineDeployments[ns] { + if deployment.Name == name { + return deployment, nil + } + } + return nil, fmt.Errorf("no machine deployment found in ns %q with name %q", ns, name) +} +func (t *target) GetMachineSet(ns, name string) (*clusterv1.MachineSet, error) { + for _, ms := range t.machineSets[ns] { + if ms.Name == name { + return ms, nil + } + } + return nil, fmt.Errorf("no machineset found with name %q in namespace %q", ns, name) +} +func (t *target) WaitForClusterV1alpha1Ready() error { + return nil +} + +type providerComponents struct { + // TODO use this then render them as YAML in the String function + // sets []*appsv1.StatefulSet + names []string +} + +func (p *providerComponents) String() string { + yamls := []string{} + for _, name := range p.names { + yamls = append(yamls, fmt.Sprintf(`kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: %s +`, name)) + } + return strings.Join(yamls, "---") +} + +func TestPivot(t *testing.T) { + pc := &providerComponents{ + names: []string{"test1", "test2"}, + } + + ns1 := "ns1" + ns2 := "ns2" + + source := newSourcer(). + WithCluster(ns1, "cluster1"). + WithMachineDeployment(ns1, "cluster1", "deployment1"). + WithMachineSet(ns1, "cluster1", "deployment1", "machineset1"). + WithMachine(ns1, "cluster1", "machineset1", "machine1"). + WithMachine(ns1, "cluster1", "machineset1", "machine2"). + WithMachineSet(ns1, "cluster1", "machinedeployment1", "machineset2"). + WithMachine(ns1, "cluster1", "machineset2", "machine3"). + WithMachineSet(ns1, "cluster1", "", "machineset3"). + WithMachine(ns1, "cluster1", "machineset3", "machine4"). + WithMachine(ns1, "cluster1", "", "machine5"). + WithMachineClass(ns1, "my-machine-class"). + WithMachineDeployment(ns1, "", "deployment2"). + WithMachineSet(ns1, "", "deployment2", "machineset4"). + WithMachine(ns1, "", "machineset4", "machine6"). + WithMachineSet(ns1, "", "", "machineset5"). + WithMachine(ns1, "", "machineset5", "machine7"). + WithMachine(ns1, "", "", "machine8"). + WithMachine(ns2, "", "", "machine9") + + expectedClusters := len(source.clusters[ns1]) + len(source.clusters[ns2]) + expectedMachineDeployments := len(source.machineDeployments[ns1]) + len(source.machineDeployments[ns2]) + expectedMachineSets := len(source.machineSets[ns1]) + len(source.machineSets[ns2]) + expectedMachines := len(source.machines[ns1]) + len(source.machines[ns2]) + expectedMachineClasses := len(source.machineClasses[ns1]) + len(source.machineClasses[ns2]) + + target := &target{ + clusters: make(map[string][]*clusterv1.Cluster), + machineDeployments: make(map[string][]*clusterv1.MachineDeployment), + machineSets: make(map[string][]*clusterv1.MachineSet), + machines: make(map[string][]*clusterv1.Machine), + machineClasses: make(map[string][]*clusterv1.MachineClass), + } + + if err := Pivot(source, target, pc.String()); err != nil { + t.Fatalf("did not expect err but got %v", err) + } + + if len(source.clusters[ns1])+len(source.clusters[ns2]) != 0 { + t.Logf("source: %v", source.clusters) + t.Logf("target: %v", target.clusters) + t.Fatal("should have deleted all capi clusters from the source k8s cluster") + } + if len(source.machineDeployments[ns1])+len(source.machineDeployments[ns2]) != 0 { + t.Logf("source: %v", source.machineDeployments) + t.Logf("target: %v", target.machineDeployments) + t.Fatal("should have deleted all machine deployments from the source k8s cluster") + } + if len(source.machineSets[ns1])+len(source.machineSets[ns2]) != 0 { + t.Logf("source: %v", source.machineSets) + t.Logf("target: %v", target.machineSets) + t.Fatal("should have deleted all machine sets from source k8s cluster") + } + if len(source.machines[ns1])+len(source.machines[ns2]) != 0 { + t.Logf("source: %v", source.machines) + t.Logf("target: %v", target.machines) + t.Fatal("should have deleted all machines from source k8s cluster") + } + if len(source.machineClasses[ns1])+len(source.machineClasses[ns2]) != 0 { + t.Logf("source: %v", source.machineClasses) + t.Logf("target: %v", target.machineClasses) + t.Fatal("should have deleted all machine classes from source k8s cluster") + } + + if len(target.clusters[ns1])+len(target.clusters[ns2]) != expectedClusters { + t.Logf("source: %v", source.clusters) + t.Logf("target: %v", target.clusters) + t.Fatal("expected clusters to pivot") + } + if len(target.machineDeployments[ns1])+len(target.machineDeployments[ns2]) != expectedMachineDeployments { + t.Logf("source: %v", source.machineDeployments) + t.Logf("target: %v", target.machineDeployments) + t.Fatal("expected machinedeployments for cluster to pivot") + } + if len(target.machineSets[ns1])+len(target.machineSets[ns2]) != expectedMachineSets { + t.Logf("source: %v", source.machineSets) + t.Logf("target: %v", target.machineSets) + for _, machineSets := range target.machineSets { + for _, ms := range machineSets { + t.Logf("machineSet: %v", ms) + } + } + t.Fatal("expected machines sets to pivot") + } + if len(target.machines[ns1])+len(target.machines[ns2]) != expectedMachines { + t.Logf("source: %v", source.machines) + t.Logf("target: %v", target.machines) + for _, machines := range target.machines { + for _, m := range machines { + t.Logf("machine: %v", m) + } + } + t.Fatal("expected machines to pivot") + } + if len(target.machineClasses[ns1])+len(target.machineClasses[ns2]) != expectedMachineClasses { + t.Logf("source: %v", source.machineClasses) + t.Logf("target: %v", target.machineClasses) + t.Fatal("expected machine classes to pivot") + } +} + +// An example of testing a failure scenario +// Override the function you want to fail with an embedded sourcer struct on a +// new type: +type waitFailSourcer struct { + *sourcer +} + +func (s *waitFailSourcer) WaitForClusterV1alpha1Ready() error { + return errors.New("failed to wait for ready cluster resources") +} +func TestWaitForV1alpha1Failure(t *testing.T) { + + w := &waitFailSourcer{ + newSourcer(), + } + err := Pivot(w, &target{}, "") + if err == nil { + t.Fatal("expected an error but got nil") + } +} diff --git a/cmd/clusterctl/testdata/delete-cluster-no-args-invalid-flag.golden b/cmd/clusterctl/testdata/delete-cluster-no-args-invalid-flag.golden index 9c0f83120dcb..dc7a15fb3611 100644 --- a/cmd/clusterctl/testdata/delete-cluster-no-args-invalid-flag.golden +++ b/cmd/clusterctl/testdata/delete-cluster-no-args-invalid-flag.golden @@ -8,7 +8,6 @@ Flags: --bootstrap-flags strings Command line flags to be passed to the chosen bootstrapper --bootstrap-type string The cluster bootstrapper to use. (default "none") --cluster string The name of the kubeconfig cluster to use - --cluster-namespace string Namespace where the cluster to be deleted resides (default "default") -h, --help help for cluster -n, --namespace string If present, the namespace scope for this CLI request -p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects, if empty the value is loaded from the cluster's configuration store. diff --git a/cmd/clusterctl/testdata/delete-cluster-no-args.golden b/cmd/clusterctl/testdata/delete-cluster-no-args.golden index 40891e9063fa..3eaea4c89bbf 100644 --- a/cmd/clusterctl/testdata/delete-cluster-no-args.golden +++ b/cmd/clusterctl/testdata/delete-cluster-no-args.golden @@ -10,7 +10,6 @@ Flags: --bootstrap-flags strings Command line flags to be passed to the chosen bootstrapper --bootstrap-type string The cluster bootstrapper to use. (default "none") --cluster string The name of the kubeconfig cluster to use - --cluster-namespace string Namespace where the cluster to be deleted resides (default "default") -h, --help help for cluster -n, --namespace string If present, the namespace scope for this CLI request -p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects, if empty the value is loaded from the cluster's configuration store. diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 3996ddede7a3..31f64846441e 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - image: gcr.io/k8s-cluster-api/cluster-api-controller:latest - name: manager + name: manager \ No newline at end of file