diff --git a/Makefile b/Makefile index c7b4e140a..8cd5302e0 100644 --- a/Makefile +++ b/Makefile @@ -16,9 +16,9 @@ IMAGE_REPOSITORY := eu.gcr.io/gardener-project/gardener/machine-controller-man IMAGE_TAG := $(shell cat VERSION) COVERPROFILE := test/output/coverprofile.out -CONTROL_NAMESPACE := default -CONTROL_KUBECONFIG := dev/target-kubeconfig.yaml -TARGET_KUBECONFIG := dev/target-kubeconfig.yaml +CONTROL_NAMESPACE := default +CONTROL_KUBECONFIG := dev/control-kubeconfig.yaml +TARGET_KUBECONFIG := dev/target-kubeconfig.yaml LEADER_ELECT := "true" MACHINE_SAFETY_OVERSHOOTING_PERIOD:=1m diff --git a/pkg/controller/alicloudmachineclass.go b/pkg/controller/alicloudmachineclass.go deleted file mode 100644 index c01d4b97d..000000000 --- a/pkg/controller/alicloudmachineclass.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// AlicloudMachineClassKind is used to identify the machineClassKind as Alicloud -const AlicloudMachineClassKind = "AlicloudMachineClass" - -func (c *controller) machineDeploymentToAlicloudMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == AlicloudMachineClassKind { - c.alicloudMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToAlicloudMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == AlicloudMachineClassKind { - c.alicloudMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToAlicloudMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - return - } - if machine.Spec.Class.Kind == AlicloudMachineClassKind { - c.alicloudMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToAlicloudMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == AlicloudMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.alicloudMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == AlicloudMachineClassKind { - c.alicloudMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == AlicloudMachineClassKind { - c.alicloudMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToAlicloudMachineClassDelete(obj interface{}) { - c.machineToAlicloudMachineClassAdd(obj) -} - -func (c *controller) alicloudMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.alicloudMachineClassQueue.Add(key) -} - -func (c *controller) alicloudMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.AlicloudMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.AlicloudMachineClass) - if new == nil || !ok { - return - } - - c.alicloudMachineClassAdd(newObj) -} - -func (c *controller) alicloudMachineClassDelete(obj interface{}) { - c.alicloudMachineClassAdd(obj) -} - -// reconcileClusterAlicloudMachineClassKey reconciles an AlicloudMachineClass due to controller resync -// or an event on the alicloudMachineClass. -func (c *controller) reconcileClusterAlicloudMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.alicloudMachineClassLister.AlicloudMachineClasses(c.namespace).Get(name) - if errors.IsNotFound(err) { - klog.Infof("%s %q: Not doing work because it has been deleted", AlicloudMachineClassKind, key) - return nil - } - if err != nil { - klog.Infof("%s %q: Unable to retrieve object from store: %v", AlicloudMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterAlicloudMachineClass(ctx, class) - if err != nil { - // Re-enqueue after a 10s window - c.enqueueAlicloudMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Get ride of this logic - c.enqueueAlicloudMachineClassAfter(class, 10*time.Minute) - } - return nil -} - -func (c *controller) reconcileClusterAlicloudMachineClass(ctx context.Context, class *v1alpha1.AlicloudMachineClass) error { - klog.V(4).Info("Start Reconciling alicloudmachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling alicloudmachineclass: ", class.Name) - - internalClass := &machine.AlicloudMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidateAlicloudMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of %s failed %s", AlicloudMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err = c.addAlicloudMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(AlicloudMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.AlicloudMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - // delete machine class finalizer if exists - return c.deleteAlicloudMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addAlicloudMachineClassFinalizers(ctx context.Context, class *v1alpha1.AlicloudMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateAlicloudMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteAlicloudMachineClassFinalizers(ctx context.Context, class *v1alpha1.AlicloudMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateAlicloudMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateAlicloudMachineClassFinalizers(ctx context.Context, class *v1alpha1.AlicloudMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.AlicloudMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.AlicloudMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating AlicloudMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the alicloudmachineclass %q", class.Name) - return err -} - -func (c *controller) enqueueAlicloudMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.alicloudMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/awsmachineclass.go b/pkg/controller/awsmachineclass.go deleted file mode 100644 index bc0b4f092..000000000 --- a/pkg/controller/awsmachineclass.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// AWSMachineClassKind is used to identify the machineClassKind as AWS -const AWSMachineClassKind = "AWSMachineClass" - -func (c *controller) machineDeploymentToAWSMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == AWSMachineClassKind { - c.awsMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToAWSMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == AWSMachineClassKind { - c.awsMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToAWSMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", obj) - return - } - if machine.Spec.Class.Kind == AWSMachineClassKind { - c.awsMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToAWSMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == AWSMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.awsMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == AWSMachineClassKind { - c.awsMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == AWSMachineClassKind { - c.awsMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToAWSMachineClassDelete(obj interface{}) { - c.machineToAWSMachineClassAdd(obj) -} - -func (c *controller) awsMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.awsMachineClassQueue.Add(key) -} - -func (c *controller) awsMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.AWSMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.AWSMachineClass) - if new == nil || !ok { - return - } - - c.awsMachineClassAdd(newObj) -} - -func (c *controller) awsMachineClassDelete(obj interface{}) { - c.awsMachineClassAdd(obj) -} - -// reconcileClusterAWSMachineClassKey reconciles an AWSMachineClass due to controller resync -// or an event on the awsMachineClass. -func (c *controller) reconcileClusterAWSMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.awsMachineClassLister.AWSMachineClasses(c.namespace).Get(name) - if errors.IsNotFound(err) { - klog.Infof("%s %q: Not doing work because it has been deleted", AWSMachineClassKind, key) - return nil - } - if err != nil { - klog.Infof("%s %q: Unable to retrieve object from store: %v", AWSMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterAWSMachineClass(ctx, class) - if err != nil { - c.enqueueAWSMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Infuture to get ride of this logic - c.enqueueAWSMachineClassAfter(class, 10*time.Minute) - } - - return nil -} - -func (c *controller) reconcileClusterAWSMachineClass(ctx context.Context, class *v1alpha1.AWSMachineClass) error { - klog.V(4).Info("Start Reconciling AWSmachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling AWSmachineclass: ", class.Name) - - internalClass := &machine.AWSMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidateAWSMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of %s failed %s", AWSMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err = c.addAWSMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(AWSMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.AWSMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - return c.deleteAWSMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addAWSMachineClassFinalizers(ctx context.Context, class *v1alpha1.AWSMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateAWSMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteAWSMachineClassFinalizers(ctx context.Context, class *v1alpha1.AWSMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateAWSMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateAWSMachineClassFinalizers(ctx context.Context, class *v1alpha1.AWSMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.AWSMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.AWSMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating AWSMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the awsmachineclass %q", class.Name) - return err -} - -func (c *controller) enqueueAWSMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.awsMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/awsmachineclass_test.go b/pkg/controller/awsmachineclass_test.go deleted file mode 100644 index ec30fdd75..000000000 --- a/pkg/controller/awsmachineclass_test.go +++ /dev/null @@ -1,441 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package controller - -import ( - "context" - "fmt" - "math" - "time" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - customfake "github.com/gardener/machine-controller-manager/pkg/fakeclient" - "github.com/gardener/machine-controller-manager/pkg/util/provider/driver" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -var ( - awsMachineClassSpec = v1alpha1.AWSMachineClassSpec{ - AMI: "test-ami", - Region: "test-region", - BlockDevices: []v1alpha1.AWSBlockDeviceMappingSpec{ - { - DeviceName: "test-device", - Ebs: v1alpha1.AWSEbsBlockDeviceSpec{ - DeleteOnTermination: new(bool), - Encrypted: false, - Iops: 100, - KmsKeyID: new(string), - SnapshotID: new(string), - VolumeSize: 50, - VolumeType: "gp2", - }, - NoDevice: "test-no", - VirtualName: "test-vname", - }, - }, - EbsOptimized: false, - IAM: v1alpha1.AWSIAMProfileSpec{ - ARN: "test-arn", - Name: "test-iam", - }, - MachineType: "m4.xlarge", - KeyName: "test-key", - Monitoring: false, - NetworkInterfaces: []v1alpha1.AWSNetworkInterfaceSpec{ - { - AssociatePublicIPAddress: new(bool), - DeleteOnTermination: new(bool), - Description: new(string), - SecurityGroupIDs: []string{"sg-0324245"}, - SubnetID: "test-subnet", - }, - }, - Tags: map[string]string{ - "kubernetes.io/cluster/name": "1", - "kubernetes.io/role/node": "1", - }, - SpotPrice: new(string), - SecretRef: &v1.SecretReference{ - Name: "test-secret", - Namespace: TestNamespace, - }, - } -) - -var _ = Describe("machineclass", func() { - Describe("#reconcileClusterAWSMachineClass", func() { - type setup struct { - machineClasses []*v1alpha1.AWSMachineClass - machines []*v1alpha1.Machine - fakeResourceActions *customfake.ResourceActions - } - type action struct { - fakeDriver *driver.FakeDriver - machineClassName string - } - type expect struct { - machineClass *v1alpha1.AWSMachineClass - err error - deleted bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.machineClasses { - machineObjects = append(machineObjects, o) - } - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - controller, trackers := createController(stop, TestNamespace, machineObjects, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, controller) - - action := data.action - machineClass, err := controller.controlMachineClient.AWSMachineClasses(TestNamespace).Get(context.TODO(), action.machineClassName, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - if data.setup.fakeResourceActions != nil { - trackers.TargetCore.SetFakeResourceActions(data.setup.fakeResourceActions, math.MaxInt32) - } - - err = controller.reconcileClusterAWSMachineClass(context.TODO(), machineClass) - if data.expect.err == nil { - Expect(err).To(Not(HaveOccurred())) - } else { - Expect(err).To(Equal(data.expect.err)) - } - - machineClass, err = controller.controlMachineClient.AWSMachineClasses(TestNamespace).Get(context.TODO(), action.machineClassName, metav1.GetOptions{}) - if data.expect.deleted { - Expect(err).To((HaveOccurred())) - } else { - Expect(err).To(Not(HaveOccurred())) - Expect(data.expect.machineClass).To(Equal(machineClass)) - } - }, - Entry( - "Add finalizer to a machine class", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: AWSMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: nil, - }, - }, - ), - Entry( - "Finalizer exists so do nothing", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: AWSMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: nil, - }, - }, - ), - Entry( - "Class deletion failure due to machines referring to it", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: AWSMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: fmt.Errorf("Retry as machine objects are still referring the machineclass"), - }, - }, - ), - Entry( - "Class deletion succeeds with removal of finalizer", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: nil, - }, - }, - ), - Entry( - "Class deletion succeeds as no finalizer exists", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: nil, - }, - }, - ), - Entry( - "Class migration has completed, set deletion timestamp", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.AWSMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - machineutils.MigratedMachineClass: "present", - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.AWSMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: awsMachineClassSpec, - }, - err: fmt.Errorf("Retry deletion as deletion timestamp is now set"), - deleted: true, - }, - }, - ), - ) - }) -}) diff --git a/pkg/controller/azuremachineclass.go b/pkg/controller/azuremachineclass.go deleted file mode 100644 index 7210e22cc..000000000 --- a/pkg/controller/azuremachineclass.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// AzureMachineClassKind is used to identify the machineClassKind as Azure -const AzureMachineClassKind = "AzureMachineClass" - -func (c *controller) machineDeploymentToAzureMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == AzureMachineClassKind { - c.azureMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToAzureMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == AzureMachineClassKind { - c.azureMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToAzureMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", obj) - return - } - if machine.Spec.Class.Kind == AzureMachineClassKind { - c.azureMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToAzureMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == AzureMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.azureMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == AzureMachineClassKind { - c.azureMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == AzureMachineClassKind { - c.azureMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToAzureMachineClassDelete(obj interface{}) { - c.machineToAzureMachineClassAdd(obj) -} - -func (c *controller) azureMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.azureMachineClassQueue.Add(key) -} - -func (c *controller) azureMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.AzureMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.AzureMachineClass) - if new == nil || !ok { - return - } - - c.azureMachineClassAdd(newObj) -} - -func (c *controller) azureMachineClassDelete(obj interface{}) { - c.azureMachineClassAdd(obj) -} - -// reconcileClusterAzureMachineClassKey reconciles an AzureMachineClass due to controller resync -// or an event on the azureMachineClass. -func (c *controller) reconcileClusterAzureMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.azureMachineClassLister.AzureMachineClasses(c.namespace).Get(name) - - if errors.IsNotFound(err) { - klog.Infof("%s %q: Not doing work because it has been deleted", AzureMachineClassKind, key) - return nil - } - if err != nil { - klog.Infof("%s %q: Unable to retrieve object from store: %v", AzureMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterAzureMachineClass(ctx, class) - if err != nil { - c.enqueueAzureMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Infuture to get ride of this logic - c.enqueueAzureMachineClassAfter(class, 10*time.Minute) - } - - return nil -} - -func (c *controller) reconcileClusterAzureMachineClass(ctx context.Context, class *v1alpha1.AzureMachineClass) error { - klog.V(4).Info("Start Reconciling Azuremachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling Azuremachineclass: ", class.Name) - - internalClass := &machine.AzureMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidateAzureMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of %s failed %s", AzureMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err = c.addAzureMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(AzureMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.AzureMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - return c.deleteAzureMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addAzureMachineClassFinalizers(ctx context.Context, class *v1alpha1.AzureMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateAzureMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteAzureMachineClassFinalizers(ctx context.Context, class *v1alpha1.AzureMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateAzureMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateAzureMachineClassFinalizers(ctx context.Context, class *v1alpha1.AzureMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.AzureMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.AzureMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating AzureMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the azuremachineclass %q", class.Name) - return err -} - -func (c *controller) enqueueAzureMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.azureMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 116a03a2a..d0911e4ba 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -56,6 +56,12 @@ const ( MachineIDAnnotation = "machine.sapcloud.io/id" // DeleteFinalizerName is the finalizer used to identify the controller acting on an object DeleteFinalizerName = "machine.sapcloud.io/machine-controller-manager" + + // MachinePriority is the annotation used to specify priority + // associated with a machine while deleting it. The less its + // priority the more likely it is to be deleted first + // Default priority for a machine is set to 3 + MachinePriority = "machinepriority.machine.sapcloud.io" ) // NewController returns a new Node controller. @@ -102,9 +108,7 @@ func NewController( machineQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machine"), machineSetQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineset"), machineDeploymentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinedeployment"), - machineSafetyOrphanVMsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyorphanvms"), machineSafetyOvershootingQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyovershooting"), - machineSafetyAPIServerQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyapiserver"), safetyOptions: safetyOptions, nodeConditions: nodeConditions, bootstrapTokenAuthExtraGroups: bootstrapTokenAuthExtraGroups, @@ -164,202 +168,6 @@ func NewController( controller.machineSetSynced = machineSetInformer.Informer().HasSynced controller.machineDeploymentSynced = machineDeploymentInformer.Informer().HasSynced - // Secret Controller Informers - secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.secretAdd, - DeleteFunc: controller.secretDelete, - }) - - openStackMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.openStackMachineClassToSecretAdd, - UpdateFunc: controller.openStackMachineClassToSecretUpdate, - DeleteFunc: controller.openStackMachineClassToSecretDelete, - }) - - gcpMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.gcpMachineClassToSecretAdd, - UpdateFunc: controller.gcpMachineClassToSecretUpdate, - DeleteFunc: controller.gcpMachineClassToSecretDelete, - }) - - azureMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.azureMachineClassToSecretAdd, - UpdateFunc: controller.azureMachineClassToSecretUpdate, - DeleteFunc: controller.azureMachineClassToSecretDelete, - }) - - alicloudMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.alicloudMachineClassToSecretAdd, - UpdateFunc: controller.alicloudMachineClassToSecretUpdate, - DeleteFunc: controller.alicloudMachineClassToSecretDelete, - }) - - awsMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.awsMachineClassToSecretAdd, - UpdateFunc: controller.awsMachineClassToSecretUpdate, - DeleteFunc: controller.awsMachineClassToSecretDelete, - }) - - packetMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.packetMachineClassToSecretAdd, - UpdateFunc: controller.packetMachineClassToSecretUpdate, - DeleteFunc: controller.packetMachineClassToSecretDelete, - }) - - // Openstack Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToOpenStackMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToOpenStackMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToOpenStackMachineClassAdd, - UpdateFunc: controller.machineToOpenStackMachineClassUpdate, - DeleteFunc: controller.machineToOpenStackMachineClassDelete, - }) - - openStackMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.openStackMachineClassAdd, - UpdateFunc: controller.openStackMachineClassUpdate, - DeleteFunc: controller.openStackMachineClassDelete, - }) - - // AWS Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToAWSMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToAWSMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToAWSMachineClassAdd, - UpdateFunc: controller.machineToAWSMachineClassUpdate, - DeleteFunc: controller.machineToAWSMachineClassDelete, - }) - - awsMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.awsMachineClassAdd, - UpdateFunc: controller.awsMachineClassUpdate, - DeleteFunc: controller.awsMachineClassDelete, - }) - - // Azure Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToAzureMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToAzureMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToAzureMachineClassAdd, - UpdateFunc: controller.machineToAzureMachineClassUpdate, - DeleteFunc: controller.machineToAzureMachineClassDelete, - }) - - azureMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.azureMachineClassAdd, - UpdateFunc: controller.azureMachineClassUpdate, - DeleteFunc: controller.azureMachineClassDelete, - }) - - // GCP Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToGCPMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToGCPMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToGCPMachineClassAdd, - UpdateFunc: controller.machineToGCPMachineClassUpdate, - DeleteFunc: controller.machineToGCPMachineClassDelete, - }) - - gcpMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.gcpMachineClassAdd, - UpdateFunc: controller.gcpMachineClassUpdate, - DeleteFunc: controller.gcpMachineClassDelete, - }) - - // Alicloud Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToAlicloudMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToAlicloudMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToAlicloudMachineClassAdd, - UpdateFunc: controller.machineToAlicloudMachineClassUpdate, - DeleteFunc: controller.machineToAlicloudMachineClassDelete, - }) - - alicloudMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.alicloudMachineClassAdd, - UpdateFunc: controller.alicloudMachineClassUpdate, - DeleteFunc: controller.alicloudMachineClassDelete, - }) - - // Packet Controller Informers - machineDeploymentInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineDeploymentToPacketMachineClassDelete, - }) - - machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.machineSetToPacketMachineClassDelete, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.machineToPacketMachineClassAdd, - UpdateFunc: controller.machineToPacketMachineClassUpdate, - DeleteFunc: controller.machineToPacketMachineClassDelete, - }) - - packetMachineClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.packetMachineClassAdd, - UpdateFunc: controller.packetMachineClassUpdate, - DeleteFunc: controller.packetMachineClassDelete, - }) - - /* Node Controller Informers - Don't remove this, saved for future use case. - nodeInformer.Informer().AddEventHandler( - cache.FilteringResourceEventHandler{ - FilterFunc: func(obj interface{}) bool { - //node := obj.(*apicorev1.Node) - return true //metav1.HasAnnotation(node.ObjectMeta, ClassAnnotation) - }, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: controller.nodeAdd, - UpdateFunc: controller.nodeUpdate, - DeleteFunc: controller.nodeDelete, - }, - }) - */ - - // Machine Controller Informers - nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.addNodeToMachine, - UpdateFunc: controller.updateNodeToMachine, - DeleteFunc: controller.deleteNodeToMachine, - }) - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.addMachine, - UpdateFunc: controller.updateMachine, - DeleteFunc: controller.deleteMachine, - }) - // MachineSet Controller Informers machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.addMachineToMachineSet, @@ -374,9 +182,6 @@ func NewController( }) // MachineDeployment Controller Informers - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - DeleteFunc: controller.deleteMachineDeployment, - }) machineSetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: controller.addMachineSetToDeployment, @@ -395,16 +200,7 @@ func NewController( // We follow the kubernetes way of reconciling the safety controller // done by adding empty key objects. We initialize it, to trigger // running of different safety loop on MCM startup. - controller.machineSafetyOrphanVMsQueue.Add("") controller.machineSafetyOvershootingQueue.Add("") - controller.machineSafetyAPIServerQueue.Add("") - - machineInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - // addMachineToSafety makes sure machine objects does not overshoot - AddFunc: controller.addMachineToSafety, - // deleteMachineToSafety makes sure that orphan VM handler is invoked - DeleteFunc: controller.deleteMachineToSafety, - }) return controller, nil } @@ -462,9 +258,7 @@ type controller struct { machineQueue workqueue.RateLimitingInterface machineSetQueue workqueue.RateLimitingInterface machineDeploymentQueue workqueue.RateLimitingInterface - machineSafetyOrphanVMsQueue workqueue.RateLimitingInterface machineSafetyOvershootingQueue workqueue.RateLimitingInterface - machineSafetyAPIServerQueue workqueue.RateLimitingInterface // syncs secretSynced cache.InformerSynced nodeSynced cache.InformerSynced @@ -497,9 +291,7 @@ func (c *controller) Run(workers int, stopCh <-chan struct{}) { defer c.machineQueue.ShutDown() defer c.machineSetQueue.ShutDown() defer c.machineDeploymentQueue.ShutDown() - defer c.machineSafetyOrphanVMsQueue.ShutDown() defer c.machineSafetyOvershootingQueue.ShutDown() - defer c.machineSafetyAPIServerQueue.ShutDown() if !cache.WaitForCacheSync(stopCh, c.secretSynced, c.nodeSynced, c.openStackMachineClassSynced, c.awsMachineClassSynced, c.azureMachineClassSynced, c.gcpMachineClassSynced, c.alicloudMachineClassSynced, c.packetMachineClassSynced, c.machineSynced, c.machineSetSynced, c.machineDeploymentSynced) { runtimeutil.HandleError(fmt.Errorf("Timed out waiting for caches to sync")) @@ -515,20 +307,9 @@ func (c *controller) Run(workers int, stopCh <-chan struct{}) { prometheus.MustRegister(c) for i := 0; i < workers; i++ { - createWorker(c.openStackMachineClassQueue, "ClusterOpenStackMachineClass", maxRetries, true, c.reconcileClusterOpenStackMachineClassKey, stopCh, &waitGroup) - createWorker(c.awsMachineClassQueue, "ClusterAWSMachineClass", maxRetries, true, c.reconcileClusterAWSMachineClassKey, stopCh, &waitGroup) - createWorker(c.azureMachineClassQueue, "ClusterAzureMachineClass", maxRetries, true, c.reconcileClusterAzureMachineClassKey, stopCh, &waitGroup) - createWorker(c.gcpMachineClassQueue, "ClusterGCPMachineClass", maxRetries, true, c.reconcileClusterGCPMachineClassKey, stopCh, &waitGroup) - createWorker(c.alicloudMachineClassQueue, "ClusterAlicloudMachineClass", maxRetries, true, c.reconcileClusterAlicloudMachineClassKey, stopCh, &waitGroup) - createWorker(c.packetMachineClassQueue, "ClusterPacketMachineClass", maxRetries, true, c.reconcileClusterPacketMachineClassKey, stopCh, &waitGroup) - createWorker(c.secretQueue, "ClusterSecret", maxRetries, true, c.reconcileClusterSecretKey, stopCh, &waitGroup) - createWorker(c.nodeQueue, "ClusterNode", maxRetries, true, c.reconcileClusterNodeKey, stopCh, &waitGroup) - createWorker(c.machineQueue, "ClusterMachine", maxRetries, true, c.reconcileClusterMachineKey, stopCh, &waitGroup) createWorker(c.machineSetQueue, "ClusterMachineSet", maxRetries, true, c.reconcileClusterMachineSet, stopCh, &waitGroup) createWorker(c.machineDeploymentQueue, "ClusterMachineDeployment", maxRetries, true, c.reconcileClusterMachineDeployment, stopCh, &waitGroup) - createWorker(c.machineSafetyOrphanVMsQueue, "ClusterMachineSafetyOrphanVMs", maxRetries, true, c.reconcileClusterMachineSafetyOrphanVMs, stopCh, &waitGroup) createWorker(c.machineSafetyOvershootingQueue, "ClusterMachineSafetyOvershooting", maxRetries, true, c.reconcileClusterMachineSafetyOvershooting, stopCh, &waitGroup) - createWorker(c.machineSafetyAPIServerQueue, "ClusterMachineAPIServer", maxRetries, true, c.reconcileClusterMachineSafetyAPIServer, stopCh, &waitGroup) } <-stopCh diff --git a/pkg/controller/controller_suite_test.go b/pkg/controller/controller_suite_test.go index a230e9191..67cca9ac4 100644 --- a/pkg/controller/controller_suite_test.go +++ b/pkg/controller/controller_suite_test.go @@ -242,15 +242,6 @@ func deepCopy(m map[string]string) map[string]string { return r } -func newMachineFromMachineSet( - machineSet *v1alpha1.MachineSet, - statusTemplate *v1alpha1.MachineStatus, - annotations map[string]string, - labels map[string]string, -) *v1alpha1.Machine { - return newMachinesFromMachineSet(1, machineSet, statusTemplate, annotations, labels)[0] -} - func newMachinesFromMachineSet( machineCount int, machineSet *v1alpha1.MachineSet, @@ -416,25 +407,6 @@ func newMachineStatus(statusTemplate *v1alpha1.MachineStatus, index int) *v1alph return r } -func newSecretReference(meta *metav1.ObjectMeta, index int) *corev1.SecretReference { - r := &corev1.SecretReference{ - Namespace: meta.Namespace, - } - - if meta.Name != "" { - r.Name = meta.Name - return r - } - - if meta.GenerateName != "" { - r.Name = fmt.Sprintf("%s-%d", meta.GenerateName, index) - return r - } - - r.Name = fmt.Sprintf("machine-%d", index) - return r -} - func createController( stop <-chan struct{}, namespace string, @@ -542,9 +514,7 @@ func createController( machineQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machine"), machineSetQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machineset"), machineDeploymentQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinedeployment"), - machineSafetyOrphanVMsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyorphanvms"), machineSafetyOvershootingQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyovershooting"), - machineSafetyAPIServerQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "machinesafetyapiserver"), expectations: NewUIDTrackingContExpectations(NewContExpectations()), recorder: record.NewBroadcaster().NewRecorder(nil, corev1.EventSource{Component: ""}), deleteMigratedMachineClass: true, diff --git a/pkg/controller/controller_utils_test.go b/pkg/controller/controller_utils_test.go index b74ab8a2a..b9108d39b 100644 --- a/pkg/controller/controller_utils_test.go +++ b/pkg/controller/controller_utils_test.go @@ -24,7 +24,6 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -296,7 +295,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -321,7 +320,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -343,7 +342,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -367,7 +366,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -391,7 +390,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -472,7 +471,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -497,7 +496,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -517,7 +516,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, @@ -542,7 +541,7 @@ var _ = Describe("#controllerUtils", func() { setup: setup{ node: newNode( 1, - &corev1.NodeSpec{}, + &v1.NodeSpec{}, nil, ), }, diff --git a/pkg/controller/gcpmachineclass.go b/pkg/controller/gcpmachineclass.go deleted file mode 100644 index d9336afce..000000000 --- a/pkg/controller/gcpmachineclass.go +++ /dev/null @@ -1,273 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// GCPMachineClassKind is used to identify the machineClassKind as GCP -const GCPMachineClassKind = "GCPMachineClass" - -func (c *controller) machineDeploymentToGCPMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == GCPMachineClassKind { - c.gcpMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToGCPMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == GCPMachineClassKind { - c.gcpMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToGCPMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", machine) - return - } - if machine.Spec.Class.Kind == GCPMachineClassKind { - c.gcpMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToGCPMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == GCPMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.gcpMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == GCPMachineClassKind { - c.gcpMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == GCPMachineClassKind { - c.gcpMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToGCPMachineClassDelete(obj interface{}) { - c.machineToGCPMachineClassAdd(obj) -} - -func (c *controller) gcpMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.gcpMachineClassQueue.Add(key) -} - -func (c *controller) gcpMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.GCPMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.GCPMachineClass) - if new == nil || !ok { - return - } - - c.gcpMachineClassAdd(newObj) -} - -func (c *controller) gcpMachineClassDelete(obj interface{}) { - c.gcpMachineClassAdd(obj) -} - -// reconcileClusterGCPMachineClassKey reconciles an GCPMachineClass due to controller resync -// or an event on the gcpMachineClass. -func (c *controller) reconcileClusterGCPMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.gcpMachineClassLister.GCPMachineClasses(c.namespace).Get(name) - - if errors.IsNotFound(err) { - klog.Infof("%s %q: Not doing work because it has been deleted", GCPMachineClassKind, key) - return nil - } - if err != nil { - klog.Infof("%s %q: Unable to retrieve object from store: %v", GCPMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterGCPMachineClass(ctx, class) - if err != nil { - c.enqueueGCPMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Infuture to get ride of this logic - c.enqueueGCPMachineClassAfter(class, 10*time.Minute) - } - - return nil -} - -func (c *controller) reconcileClusterGCPMachineClass(ctx context.Context, class *v1alpha1.GCPMachineClass) error { - klog.V(4).Info("Start Reconciling GCPmachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling GCPmachineclass: ", class.Name) - - internalClass := &machine.GCPMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidateGCPMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of %s failed %s", GCPMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err = c.addGCPMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(GCPMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.GCPMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - return c.deleteGCPMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addGCPMachineClassFinalizers(ctx context.Context, class *v1alpha1.GCPMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateGCPMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteGCPMachineClassFinalizers(ctx context.Context, class *v1alpha1.GCPMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateGCPMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateGCPMachineClassFinalizers(ctx context.Context, class *v1alpha1.GCPMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.GCPMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.GCPMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating GCPMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the gcpmachineclass %q", class.Name) - return err -} - -func (c *controller) enqueueGCPMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.gcpMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/gcpmachineclass_test.go b/pkg/controller/gcpmachineclass_test.go deleted file mode 100644 index 4e2d697d8..000000000 --- a/pkg/controller/gcpmachineclass_test.go +++ /dev/null @@ -1,927 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package controller - -import ( - "context" - "fmt" - "math" - "time" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - customfake "github.com/gardener/machine-controller-manager/pkg/fakeclient" - "github.com/gardener/machine-controller-manager/pkg/util/provider/driver" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -const ( - TestMachineName = "test-machine" - TestMachineClassName = "test-mc" - TestNamespace = "test-ns" - TestSecret = "test-secret" -) - -var _ = Describe("machineclass", func() { - Describe("#reconcileClusterGCPMachineClass", func() { - type setup struct { - machineClasses []*v1alpha1.GCPMachineClass - machines []*v1alpha1.Machine - fakeResourceActions *customfake.ResourceActions - } - type action struct { - fakeDriver *driver.FakeDriver - machineClassName string - } - type expect struct { - machineClass *v1alpha1.GCPMachineClass - err error - deleted bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.machineClasses { - machineObjects = append(machineObjects, o) - } - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - controller, trackers := createController(stop, TestNamespace, machineObjects, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, controller) - - action := data.action - machineClass, err := controller.controlMachineClient.GCPMachineClasses(TestNamespace).Get(context.TODO(), action.machineClassName, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - if data.setup.fakeResourceActions != nil { - trackers.TargetCore.SetFakeResourceActions(data.setup.fakeResourceActions, math.MaxInt32) - } - - err = controller.reconcileClusterGCPMachineClass(context.TODO(), machineClass) - if data.expect.err == nil { - Expect(err).To(Not(HaveOccurred())) - } else { - Expect(err).To(Equal(data.expect.err)) - } - - machineClass, err = controller.controlMachineClient.GCPMachineClasses(TestNamespace).Get(context.TODO(), action.machineClassName, metav1.GetOptions{}) - if data.expect.deleted { - Expect(err).To((HaveOccurred())) - } else { - Expect(err).To(Not(HaveOccurred())) - Expect(data.expect.machineClass).To(Equal(machineClass)) - } - }, - Entry( - "Add finalizer to a machine class", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: GCPMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: nil, - }, - }, - ), - Entry( - "Finalizer exists so do nothing", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: GCPMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: nil, - }, - }, - ), - Entry( - "Class deletion failure due to machines referring to it", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: TestMachineName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.MachineSpec{ - Class: v1alpha1.ClassSpec{ - Name: TestMachineClassName, - Kind: GCPMachineClassKind, - }, - ProviderID: "", - NodeTemplateSpec: v1alpha1.NodeTemplateSpec{}, - MachineConfiguration: &v1alpha1.MachineConfiguration{}, - }, - Status: v1alpha1.MachineStatus{}, - }, - }, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: fmt.Errorf("Retry as machine objects are still referring the machineclass"), - }, - }, - ), - Entry( - "Class deletion succeeds with removal of finalizer", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: nil, - }, - }, - ), - Entry( - "Class deletion succeeds as no finalizer exists", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: nil, - }, - }, - ), - Entry( - "Class migration has completed, set deletion timestamp", - &data{ - setup: setup{ - machineClasses: []*v1alpha1.GCPMachineClass{ - { - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - machineutils.MigratedMachineClass: "present", - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - }, - machines: []*v1alpha1.Machine{}, - fakeResourceActions: &customfake.ResourceActions{}, - }, - action: action{ - fakeDriver: &driver.FakeDriver{}, - machineClassName: TestMachineClassName, - }, - expect: expect{ - machineClass: &v1alpha1.GCPMachineClass{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: &metav1.Time{ - Time: time.Time{}, - }, - Finalizers: []string{DeleteFinalizerName}, - Name: TestMachineClassName, - Namespace: TestNamespace, - }, - TypeMeta: metav1.TypeMeta{}, - Spec: v1alpha1.GCPMachineClassSpec{ - CanIpForward: false, - DeletionProtection: false, - Description: new(string), - Disks: []*v1alpha1.GCPDisk{ - { - AutoDelete: new(bool), - Boot: true, - SizeGb: 50, - Type: "pd-ssd", - Image: "test-image", - }, - }, - Labels: map[string]string{}, - MachineType: "test-type", - Metadata: []*v1alpha1.GCPMetadata{}, - NetworkInterfaces: []*v1alpha1.GCPNetworkInterface{ - { - DisableExternalIP: false, - Network: "test-network", - Subnetwork: "test-subnetwork", - }, - }, - Scheduling: v1alpha1.GCPScheduling{ - AutomaticRestart: false, - OnHostMaintenance: "MIGRATE", - Preemptible: false, - }, - SecretRef: &v1.SecretReference{ - Name: TestSecret, - Namespace: TestNamespace, - }, - ServiceAccounts: []v1alpha1.GCPServiceAccount{ - { - Email: "test@test.com", - Scopes: []string{"test-scope"}, - }, - }, - Tags: []string{ - "kubernetes-io-cluster-test", - "kubernetes-io-role-node", - }, - Region: "test-region", - Zone: "test-zone", - }, - }, - err: fmt.Errorf("Retry deletion as deletion timestamp is now set"), - deleted: true, - }, - }, - ), - ) - }) -}) diff --git a/pkg/controller/machine.go b/pkg/controller/machine.go index b47ec8945..98e474c6f 100644 --- a/pkg/controller/machine.go +++ b/pkg/controller/machine.go @@ -18,810 +18,17 @@ limitations under the License. package controller import ( - "bytes" "context" - "errors" - "fmt" - "math" - "strings" "time" "k8s.io/klog/v2" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - apiequality "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/driver" - utiltime "github.com/gardener/machine-controller-manager/pkg/util/time" ) -const ( - // MachinePriority is the annotation used to specify priority - // associated with a machine while deleting it. The less its - // priority the more likely it is to be deleted first - // Default priority for a machine is set to 3 - MachinePriority = "machinepriority.machine.sapcloud.io" - - // MachineEnqueueRetryPeriod period after which a machine object is re-enqueued upon creation or deletion failures. - MachineEnqueueRetryPeriod = 2 * time.Minute -) - -/* - SECTION - Machine controller - Machine add, update, delete watches -*/ -func (c *controller) addMachine(obj interface{}) { - klog.V(4).Infof("Adding machine object") - c.enqueueMachine(obj) -} - -func (c *controller) updateMachine(oldObj, newObj interface{}) { - klog.V(4).Info("Updating machine object") - c.enqueueMachine(newObj) -} - -func (c *controller) deleteMachine(obj interface{}) { - klog.V(4).Info("Deleting machine object") - c.enqueueMachine(obj) -} - -func (c *controller) enqueueMachine(obj interface{}) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - - machine := obj.(*v1alpha1.Machine) - switch machine.Spec.Class.Kind { - case AlicloudMachineClassKind, AWSMachineClassKind, AzureMachineClassKind, GCPMachineClassKind, OpenStackMachineClassKind, PacketMachineClassKind: - // Checking if machineClass is to be processed by MCM, and then only enqueue the machine object - klog.V(4).Infof("Adding machine object to the queue %q", key) - c.machineQueue.Add(key) - default: - klog.V(4).Infof("ClassKind %q not found. Machine maybe be processed by external controller", machine.Spec.Class.Kind) - } -} - -func (c *controller) enqueueMachineAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - - machine := obj.(*v1alpha1.Machine) - - switch machine.Spec.Class.Kind { - case AlicloudMachineClassKind, AWSMachineClassKind, AzureMachineClassKind, GCPMachineClassKind, OpenStackMachineClassKind, PacketMachineClassKind: - // Checking if machineClass is to be processed by MCM, and then only enqueue the machine object - klog.V(4).Infof("Adding machine object to the queue %q after %s", key, after) - c.machineQueue.AddAfter(key, after) - default: - klog.V(4).Infof("ClassKind %q not found. Machine maybe be processed by external controller", machine.Spec.Class.Kind) - } -} - -func (c *controller) reconcileClusterMachineKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - machine, err := c.machineLister.Machines(c.namespace).Get(name) - if apierrors.IsNotFound(err) { - klog.V(4).Infof("Machine %q: Not doing work because it is not found", key) - return nil - } - - if err != nil { - klog.Errorf("ClusterMachine %q: Unable to retrieve object from store: %v", key, err) - return err - } - - return c.reconcileClusterMachine(ctx, machine) -} - -func (c *controller) reconcileClusterMachine(ctx context.Context, machine *v1alpha1.Machine) error { - klog.V(4).Info("Start Reconciling machine: ", machine.Name) - defer func() { - c.enqueueMachineAfter(machine, 10*time.Minute) - klog.V(4).Info("Stop Reconciling machine: ", machine.Name) - }() - - if c.safetyOptions.MachineControllerFrozen && machine.DeletionTimestamp == nil { - message := "Machine controller has frozen. Retrying reconcile after 10 minutes" - klog.V(3).Info(message) - return errors.New(message) - } - - if !shouldReconcileMachine(machine, time.Now()) { - return nil - } - - // Validate Machine - internalMachine := &machineapi.Machine{} - err := c.internalExternalScheme.Convert(machine, internalMachine, nil) - if err != nil { - return err - } - validationerr := validation.ValidateMachine(internalMachine) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of Machine failed %s", validationerr.ToAggregate().Error()) - return nil - } - - // Validate MachineClass - MachineClass, secretData, err := c.validateMachineClass(&machine.Spec.Class) - if err != nil || secretData == nil { - c.enqueueMachineAfter(machine, MachineEnqueueRetryPeriod) - return nil - } - - driver := driver.NewDriver(machine.Spec.ProviderID, secretData, machine.Spec.Class.Kind, MachineClass, machine.Name) - actualProviderID, err := driver.GetExisting() - if err != nil { - return err - } else if actualProviderID == "fake" { - klog.Warning("Fake driver type") - return nil - } - - machine, err = c.machineLister.Machines(machine.Namespace).Get(machine.Name) - if err != nil { - klog.Errorf("Could not fetch machine object %s", err) - if apierrors.IsNotFound(err) { - // Ignore the error "Not Found" - return nil - } - - return err - } - - machine, err = c.updateMachineState(ctx, machine) - if err != nil { - klog.Errorf("Could not update machine state for: %s", machine.Name) - return err - } - - // Sync nodeTemplate between machine and node-objects. - node, _ := c.nodeLister.Get(machine.Status.Node) - if node != nil { - err = c.syncMachineNodeTemplates(ctx, machine) - if err != nil { - klog.Errorf("Could not update nodeTemplate for machine %s err: %q", machine.Name, err) - return err - } - } - - if machine.DeletionTimestamp != nil { - // Processing of delete event - if err := c.machineDelete(ctx, machine, driver); err != nil { - c.enqueueMachineAfter(machine, MachineEnqueueRetryPeriod) - return nil - } - } else if machine.Status.CurrentStatus.TimeoutActive { - // Processing machine - c.checkMachineTimeout(ctx, machine) - } else { - // Processing of create or update event - c.addMachineFinalizers(ctx, machine) - - if machine.Status.CurrentStatus.Phase == v1alpha1.MachineFailed { - return nil - } else if actualProviderID == "" { - if err := c.machineCreate(ctx, machine, driver); err != nil { - c.enqueueMachineAfter(machine, MachineEnqueueRetryPeriod) - return nil - } - } else if actualProviderID != machine.Spec.ProviderID { - if err := c.machineUpdate(ctx, machine, actualProviderID); err != nil { - return err - } - } - } - - return nil -} - -/* - SECTION - Machine controller - nodeToMachine -*/ -func (c *controller) addNodeToMachine(obj interface{}) { - node := obj.(*corev1.Node) - if node == nil { - klog.Errorf("Couldn't convert to node from object") - return - } - - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - - machine, err := c.getMachineFromNode(key) - if err != nil { - klog.Errorf("Couldn't fetch machine %s, Error: %s", key, err) - return - } else if machine == nil { - return - } - - if machine.Status.CurrentStatus.Phase != v1alpha1.MachineCrashLoopBackOff && nodeConditionsHaveChanged(machine.Status.Conditions, node.Status.Conditions) { - klog.V(4).Infof("Enqueue machine object %q as backing node's conditions have changed", machine.Name) - c.enqueueMachine(machine) - } -} - -func (c *controller) updateNodeToMachine(oldObj, newObj interface{}) { - c.addNodeToMachine(newObj) -} - -func (c *controller) deleteNodeToMachine(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - - machine, err := c.getMachineFromNode(key) - if err != nil { - klog.Errorf("Couldn't fetch machine %s, Error: %s", key, err) - return - } else if machine == nil { - return - } - - c.enqueueMachine(machine) -} - -/* - SECTION - NodeToMachine operations -*/ - -func (c *controller) getMachineFromNode(nodeName string) (*v1alpha1.Machine, error) { - var ( - list = []string{nodeName} - selector = labels.NewSelector() - req, _ = labels.NewRequirement("node", selection.Equals, list) - ) - - selector = selector.Add(*req) - machines, _ := c.machineLister.List(selector) - - if len(machines) > 1 { - return nil, errors.New("Multiple machines matching node") - } else if len(machines) < 1 { - return nil, nil - } - - return machines[0], nil -} - -func (c *controller) updateMachineState(ctx context.Context, machine *v1alpha1.Machine) (*v1alpha1.Machine, error) { - nodeName := machine.Status.Node - - if nodeName == "" { - // Check if any existing node-object can be adopted. - nodeList, err := c.nodeLister.List(labels.Everything()) - if err != nil { - klog.Errorf("Could not list the nodes due to error: %v", err) - return machine, err - } - for _, node := range nodeList { - nID, mID := decodeMachineID(node.Spec.ProviderID), decodeMachineID(machine.Spec.ProviderID) - if nID == "" { - continue - } - - if nID == mID { - klog.V(2).Infof("Adopting the node object %s for machine %s", node.Name, machine.Name) - nodeName = node.Name - clone := machine.DeepCopy() - clone.Status.Node = nodeName - clone, err = c.controlMachineClient.Machines(clone.Namespace).UpdateStatus(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Errorf("Could not update status of the machine-object %s due to error %v", machine.Name, err) - return machine, err - } - break - } - } - // Couldnt adopt any node-object. - if nodeName == "" { - // There are no objects mapped to this machine object - // Hence node status need not be propogated to machine object - return machine, nil - } - } - - node, err := c.nodeLister.Get(nodeName) - if err != nil && apierrors.IsNotFound(err) { - // Node object is not found - - if len(machine.Status.Conditions) > 0 && - machine.Status.CurrentStatus.Phase == v1alpha1.MachineRunning { - // If machine has conditions on it, - // and corresponding node object went missing - // and machine is still healthy - msg := fmt.Sprintf( - "Node object went missing. Machine %s is unhealthy - changing MachineState to Unknown", - machine.Name, - ) - klog.Warning(msg) - - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineUnknown, - TimeoutActive: true, - LastUpdateTime: metav1.Now(), - } - lastOperation := v1alpha1.LastOperation{ - Description: msg, - State: v1alpha1.MachineStateProcessing, - Type: v1alpha1.MachineOperationHealthCheck, - LastUpdateTime: metav1.Now(), - } - clone, err := c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - if err != nil { - klog.Errorf("Machine updated failed for %s, Error: %q", machine.Name, err) - return machine, err - } - return clone, nil - } - // Cannot update node status as node doesn't exist - // Hence returning - return machine, nil - } else if err != nil { - // Any other types of errors while fetching node object - klog.Errorf("Could not fetch node object for machine %s", machine.Name) - return machine, err - } - - machine, err = c.updateMachineConditions(ctx, machine, node.Status.Conditions) - if err != nil { - return machine, err - } - - clone := machine.DeepCopy() - if clone.Labels == nil { - clone.Labels = make(map[string]string) - } - - if n := clone.Labels["node"]; n == "" { - clone.Labels["node"] = machine.Status.Node - machine, err = c.controlMachineClient.Machines(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warningf("Machine update failed. Retrying, error: %s", err) - return machine, err - } - } - - return machine, nil -} - -/* - SECTION - Machine operations - Create, Update, Delete -*/ - -func (c *controller) machineCreate(ctx context.Context, machine *v1alpha1.Machine, driver driver.Driver) error { - klog.V(2).Infof("Creating machine %q, please wait!", machine.Name) - var actualProviderID, nodeName string - - err := c.addBootstrapTokenToUserData(ctx, machine.Name, driver) - if err != nil { - klog.Errorf("Error while creating bootstrap token for machine %s: %s", machine.Name, err.Error()) - lastOperation := v1alpha1.LastOperation{ - Description: "MCM message - " + err.Error(), - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineFailed, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - return err - } - // Before actually creating the machine, we should once check and adopt if the virtual machine already exists. - VMList, err := driver.GetVMs("") - if err != nil { - - klog.Errorf("Error while listing machine %s: %s", machine.Name, err.Error()) - lastOperation := v1alpha1.LastOperation{ - Description: "Cloud provider message - " + err.Error(), - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineCrashLoopBackOff, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - - // Delete the bootstrap token - if err := c.deleteBootstrapToken(ctx, machine.Name); err != nil { - klog.Warning(err) - } - - klog.Errorf("Failed to list VMs before creating machine %q %+v", machine.Name, err) - return err - } - for providerID, machineName := range VMList { - if machineName == machine.Name { - klog.V(2).Infof("Adopted an existing VM %s for machine object %s.", providerID, machineName) - actualProviderID = providerID - } - } - if actualProviderID == "" { - actualProviderID, nodeName, err = driver.Create() - } - - if err != nil { - klog.Errorf("Error while creating machine %s: %s", machine.Name, err.Error()) - lastOperation := v1alpha1.LastOperation{ - Description: "Cloud provider message - " + err.Error(), - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineCrashLoopBackOff, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - - // Delete the bootstrap token - if err := c.deleteBootstrapToken(ctx, machine.Name); err != nil { - klog.Warning(err) - } - - return err - } - klog.V(2).Infof("Created/Adopted machine: %q, MachineID: %s", machine.Name, actualProviderID) - - for { - machineName := machine.Name - // Get the latest version of the machine so that we can avoid conflicts - machine, err := c.controlMachineClient.Machines(machine.Namespace).Get(ctx, machine.Name, metav1.GetOptions{}) - if err != nil { - - if apierrors.IsNotFound(err) { - klog.Infof("Machine %q not found on APIServer anymore. Deleting created (orphan) VM", machineName) - - if err = driver.Delete(actualProviderID); err != nil { - klog.Errorf( - "Deletion failed for orphan machine %q with provider-ID %q: %s", - machine.Name, - actualProviderID, - err, - ) - } - - // Return with error - return fmt.Errorf("Couldn't find machine object, hence deleted orphan VM") - } - - klog.Warningf("Machine GET failed for %q. Retrying, error: %s", machineName, err) - continue - } - - lastOperation := v1alpha1.LastOperation{ - Description: "Creating machine on cloud provider", - State: v1alpha1.MachineStateProcessing, - Type: v1alpha1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachinePending, - TimeoutActive: true, - LastUpdateTime: metav1.Now(), - } - - clone := machine.DeepCopy() - if clone.Labels == nil { - clone.Labels = make(map[string]string) - } - clone.Labels["node"] = nodeName - - if clone.Annotations == nil { - clone.Annotations = make(map[string]string) - } - if clone.Annotations[MachinePriority] == "" { - clone.Annotations[MachinePriority] = "3" - } - clone.Spec.ProviderID = actualProviderID - machine, err = c.controlMachineClient.Machines(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warningf("Machine UPDATE failed for %q. Retrying, error: %s", machineName, err) - continue - } - - clone = machine.DeepCopy() - clone.Status.Node = nodeName - clone.Status.LastOperation = lastOperation - clone.Status.CurrentStatus = currentStatus - _, err = c.controlMachineClient.Machines(clone.Namespace).UpdateStatus(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warningf("Machine/status UPDATE failed for %q. Retrying, error: %s", machineName, err) - continue - } - // Update went through, exit out of infinite loop - break - } - - return nil -} - -func (c *controller) machineUpdate(ctx context.Context, machine *v1alpha1.Machine, actualProviderID string) error { - klog.V(2).Infof("Setting MachineId of %s to %s", machine.Name, actualProviderID) - - for { - machine, err := c.controlMachineClient.Machines(machine.Namespace).Get(ctx, machine.Name, metav1.GetOptions{}) - if err != nil { - klog.Errorf("Could not fetch machine object while setting up MachineId %s for Machine %s due to error %s", actualProviderID, machine.Name, err) - return err - } - - clone := machine.DeepCopy() - clone.Spec.ProviderID = actualProviderID - machine, err = c.controlMachineClient.Machines(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warningf("Machine update failed. Retrying, error: %s", err) - continue - } - - clone = machine.DeepCopy() - lastOperation := v1alpha1.LastOperation{ - Description: "Updated provider ID", - State: v1alpha1.MachineStateSuccessful, - Type: v1alpha1.MachineOperationUpdate, - LastUpdateTime: metav1.Now(), - } - clone.Status.LastOperation = lastOperation - _, err = c.controlMachineClient.Machines(clone.Namespace).UpdateStatus(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warningf("Machine/status update failed. Retrying, error: %s", err) - continue - } - // Update went through, exit out of infinite loop - break - } - - return nil -} - -func (c *controller) machineDelete(ctx context.Context, machine *v1alpha1.Machine, driver driver.Driver) error { - var err error - nodeName := machine.Status.Node - - if finalizers := sets.NewString(machine.Finalizers...); finalizers.Has(DeleteFinalizerName) { - klog.V(2).Infof("Deleting Machine %q", machine.Name) - var ( - forceDeletePods = false - forceDeleteMachine = false - timeOutOccurred = false - maxEvictRetries = int32(math.Min(float64(*c.getEffectiveMaxEvictRetries(machine)), c.getEffectiveDrainTimeout(machine).Seconds()/PodEvictionRetryInterval.Seconds())) - pvDetachTimeOut = c.safetyOptions.PvDetachTimeout.Duration - timeOutDuration = c.getEffectiveDrainTimeout(machine).Duration - forceDeleteLabelPresent = machine.Labels["force-deletion"] == "True" - ) - - if machine.Status.CurrentStatus.Phase != v1alpha1.MachineTerminating { - lastOperation := v1alpha1.LastOperation{ - Description: "Deleting machine from cloud provider", - State: v1alpha1.MachineStateProcessing, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineTerminating, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - - err = c.deleteBootstrapToken(ctx, machine.Name) - if err != nil { - klog.Warning(err) - } - - machine, err = c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - if err != nil && apierrors.IsNotFound(err) { - // Object no longer exists and has been deleted - klog.Warning(err) - return nil - } else if err != nil { - // Any other type of errors - klog.Error(err) - return err - } - } - - timeOutOccurred = utiltime.HasTimeOutOccurred(*machine.DeletionTimestamp, timeOutDuration) - - if forceDeleteLabelPresent || timeOutOccurred { - // To perform forceful machine drain/delete either one of the below conditions must be satified - // 1. force-deletion: "True" label must be present - // 2. Deletion operation is more than drain-timeout minutes old - forceDeleteMachine = true - forceDeletePods = true - timeOutDuration = 1 * time.Minute - maxEvictRetries = 1 - - klog.V(2).Infof( - "Force delete/drain has been triggerred for machine %q due to Label:%t, timeout:%t", - machine.Name, - forceDeleteLabelPresent, - timeOutOccurred, - ) - } else { - klog.V(2).Infof( - "Normal delete/drain has been triggerred for machine %q with drain-timeout:%v & maxEvictRetries:%d", - machine.Name, - timeOutDuration, - maxEvictRetries, - ) - } - - // If machine was created on the cloud provider - machineID, _ := driver.GetExisting() - - // update node with the machine's state prior to termination - if nodeName != "" && machineID != "" { - if err = c.UpdateNodeTerminationCondition(ctx, machine); err != nil { - if forceDeleteMachine { - klog.Warningf("failed to update node conditions: %v. However, since it's a force deletion shall continue deletion of VM.", err) - } else { - klog.Error(err) - return err - } - } - } - - if machineID != "" && nodeName != "" { - // Begin drain logic only when the nodeName & providerID exist's for the machine - buf := bytes.NewBuffer([]byte{}) - errBuf := bytes.NewBuffer([]byte{}) - - drainOptions := NewDrainOptions( - c.targetCoreClient, - timeOutDuration, - maxEvictRetries, - pvDetachTimeOut, - nodeName, - 5*time.Minute, - -1, - forceDeletePods, - true, - true, - true, - buf, - errBuf, - driver, - c.pvcLister, - c.pvLister, - ) - err = drainOptions.RunDrain(ctx) - if err == nil { - // Drain successful - klog.V(2).Infof("Drain successful for machine %q. \nBuf:%v \nErrBuf:%v", machine.Name, buf, errBuf) - - } else if err != nil && forceDeleteMachine { - // Drain failed on force deletion - klog.Warningf("Drain failed for machine %q. However, since it's a force deletion shall continue deletion of VM. \nBuf:%v \nErrBuf:%v \nErr-Message:%v", machine.Name, buf, errBuf, err) - - } else { - // Drain failed on normal (non-force) deletion, return error for retry - lastOperation := v1alpha1.LastOperation{ - Description: "Drain failed - " + err.Error(), - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, machine.Status.CurrentStatus) - - klog.Warningf("Drain failed for machine %q. \nBuf:%v \nErrBuf:%v \nErr-Message:%v", machine.Name, buf, errBuf, err) - return err - } - - err = driver.Delete(machineID) - } else { - klog.V(2).Infof("Machine %q on deletion doesn't have a providerID attached to it. Checking for any VM linked to this machine object.", machine.Name) - // As MachineID is missing, we should check once if actual VM was created but MachineID was not updated on machine-object. - // We list VMs and check if any one them map with the given machine-object. - var VMList map[string]string - VMList, err = driver.GetVMs("") - if err == nil { - for providerID, machineName := range VMList { - if machineName == machine.Name { - klog.V(2).Infof("Deleting the VM %s backing the machine-object %s.", providerID, machine.Name) - err = driver.Delete(providerID) - if err != nil { - klog.Errorf("Error deleting the VM %s backing the machine-object %s due to error %v", providerID, machine.Name, err) - // Not returning error so that status on the machine object can be updated in the next step if errored. - } else { - klog.V(2).Infof("VM %s backing the machine-object %s is deleted successfully", providerID, machine.Name) - } - } - } - } else { - klog.Errorf("Failed to list VMs while deleting the machine %q %v", machine.Name, err) - } - } - - if err != nil { - // When machine deletion fails - klog.Errorf("Deletion failed for machine %q: %s", machine.Name, err) - - lastOperation := v1alpha1.LastOperation{ - Description: "Cloud provider message - " + err.Error(), - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - } - currentStatus := v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineTerminating, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - return err - } - - if nodeName != "" { - // Delete node object - err = c.targetCoreClient.CoreV1().Nodes().Delete(ctx, nodeName, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - // If its an error, and anyother error than object not found - message := fmt.Sprintf("Deletion of Node Object %q failed due to error: %s", nodeName, err) - lastOperation := v1alpha1.LastOperation{ - Description: message, - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - } - c.updateMachineStatus(ctx, machine, lastOperation, machine.Status.CurrentStatus) - klog.Errorf(message) - return err - } - } - - // Remove finalizers from machine object - c.deleteMachineFinalizers(ctx, machine) - - // Delete machine object - err = c.controlMachineClient.Machines(machine.Namespace).Delete(ctx, machine.Name, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - // If its an error, and anyother error than object not found - klog.Errorf("Deletion of Machine Object %q failed due to error: %s", machine.Name, err) - return err - } - - klog.V(2).Infof("Machine %q deleted successfully", machine.Name) - } - return nil -} - /* SECTION Update machine object @@ -874,319 +81,3 @@ func isMachineStatusEqual(s1, s2 v1alpha1.MachineStatus) bool { return apiequality.Semantic.DeepEqual(s1Copy.LastOperation, s2Copy.LastOperation) && apiequality.Semantic.DeepEqual(s1Copy.CurrentStatus, s2Copy.CurrentStatus) } - -func (c *controller) updateMachineConditions(ctx context.Context, machine *v1alpha1.Machine, conditions []v1.NodeCondition) (*v1alpha1.Machine, error) { - - var ( - msg string - lastOperationType v1alpha1.MachineOperationType - objectRequiresUpdate bool - ) - - // Get the latest version of the machine so that we can avoid conflicts - machine, err := c.controlMachineClient.Machines(machine.Namespace).Get(ctx, machine.Name, metav1.GetOptions{}) - if err != nil { - return machine, err - } - - clone := machine.DeepCopy() - - if nodeConditionsHaveChanged(clone.Status.Conditions, conditions) { - clone.Status.Conditions = conditions - objectRequiresUpdate = true - } - - if clone.Status.CurrentStatus.Phase == v1alpha1.MachineTerminating { - // If machine is already in terminating state, don't update health status - - } else if !c.isHealthy(clone) && clone.Status.CurrentStatus.Phase == v1alpha1.MachineRunning { - // If machine is not healthy, and current state is running, - // change the machinePhase to unknown and activate health check timeout - msg = fmt.Sprintf("Machine %s is unhealthy - changing MachineState to Unknown", clone.Name) - klog.Warning(msg) - - clone.Status.CurrentStatus = v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineUnknown, - TimeoutActive: true, - LastUpdateTime: metav1.Now(), - } - clone.Status.LastOperation = v1alpha1.LastOperation{ - Description: msg, - State: v1alpha1.MachineStateProcessing, - Type: v1alpha1.MachineOperationHealthCheck, - LastUpdateTime: metav1.Now(), - } - objectRequiresUpdate = true - - } else if c.isHealthy(clone) && clone.Status.CurrentStatus.Phase != v1alpha1.MachineRunning { - // If machine is healhy and current machinePhase is not running. - // indicates that the machine is not healthy and status needs to be updated. - - if clone.Status.LastOperation.Type == v1alpha1.MachineOperationCreate && - clone.Status.LastOperation.State != v1alpha1.MachineStateSuccessful { - // When machine creation went through - msg = fmt.Sprintf("Machine %s successfully joined the cluster", clone.Name) - // Delete the bootstrap token - err = c.deleteBootstrapToken(ctx, clone.Name) - if err != nil { - klog.Warning(err) - } - - lastOperationType = v1alpha1.MachineOperationCreate - } else { - // Machine rejoined the cluster after a healthcheck - msg = fmt.Sprintf("Machine %s successfully re-joined the cluster", clone.Name) - lastOperationType = v1alpha1.MachineOperationHealthCheck - } - klog.V(2).Infof(msg) - - // Machine is ready and has joined/re-joined the cluster - clone.Status.LastOperation = v1alpha1.LastOperation{ - Description: msg, - State: v1alpha1.MachineStateSuccessful, - Type: lastOperationType, - LastUpdateTime: metav1.Now(), - } - clone.Status.CurrentStatus = v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineRunning, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - objectRequiresUpdate = true - - } - - if objectRequiresUpdate { - clone, err = c.controlMachineClient.Machines(clone.Namespace).UpdateStatus(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - // Keep retrying until update goes through - klog.Warningf("Updated failed, retrying, error: %q", err) - return c.updateMachineConditions(ctx, machine, conditions) - } - - return clone, nil - } - - return machine, nil -} - -func (c *controller) updateMachineFinalizers(ctx context.Context, machine *v1alpha1.Machine, finalizers []string) { - // Get the latest version of the machine so that we can avoid conflicts - machine, err := c.controlMachineClient.Machines(machine.Namespace).Get(ctx, machine.Name, metav1.GetOptions{}) - if err != nil { - return - } - - clone := machine.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.Machines(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - // Keep retrying until update goes through - klog.Warningf("Warning: Updated failed, retrying, error: %q", err) - c.updateMachineFinalizers(ctx, machine, finalizers) - } -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addMachineFinalizers(ctx context.Context, machine *v1alpha1.Machine) { - clone := machine.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - c.updateMachineFinalizers(ctx, clone, finalizers.List()) - } -} - -func (c *controller) deleteMachineFinalizers(ctx context.Context, machine *v1alpha1.Machine) { - clone := machine.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - c.updateMachineFinalizers(ctx, clone, finalizers.List()) - } -} - -/* - SECTION - Helper Functions -*/ -func (c *controller) isHealthy(machine *v1alpha1.Machine) bool { - numOfConditions := len(machine.Status.Conditions) - - if numOfConditions == 0 { - // Kubernetes node object for this machine hasn't been received - return false - } - - for _, condition := range machine.Status.Conditions { - if condition.Type == v1.NodeReady && condition.Status != v1.ConditionTrue { - // If Kubelet is not ready - return false - } - conditions := strings.Split(*c.getEffectiveNodeConditions(machine), ",") - for _, c := range conditions { - if string(condition.Type) == c && condition.Status != v1.ConditionFalse { - return false - } - } - } - return true -} - -func (c *controller) getSecret(ref *v1.SecretReference, machineClassName string) (*v1.Secret, error) { - secretRef, err := c.secretLister.Secrets(ref.Namespace).Get(ref.Name) - if apierrors.IsNotFound(err) { - klog.V(3).Infof("No secret %q: found for MachineClass %q", ref, machineClassName) - return nil, err - } - if err != nil { - klog.Errorf("Unable get secret %q for MachineClass %q: %v", ref, machineClassName, err) - return nil, err - } - return secretRef, err -} - -func (c *controller) checkMachineTimeout(ctx context.Context, machine *v1alpha1.Machine) { - - // If machine phase is running already, ignore this loop - if machine.Status.CurrentStatus.Phase != v1alpha1.MachineRunning { - - var ( - description string - lastOperation v1alpha1.LastOperation - currentStatus v1alpha1.CurrentStatus - timeOutDuration time.Duration - ) - - checkCreationTimeout := machine.Status.CurrentStatus.Phase == v1alpha1.MachinePending - sleepTime := 1 * time.Minute - - if checkCreationTimeout { - timeOutDuration = c.getEffectiveCreationTimeout(machine).Duration - } else { - timeOutDuration = c.getEffectiveHealthTimeout(machine).Duration - } - - // Timeout value obtained by subtracting last operation with expected time out period - timeOut := metav1.Now().Add(-timeOutDuration).Sub(machine.Status.CurrentStatus.LastUpdateTime.Time) - if timeOut > 0 { - // Machine health timeout occurs while joining or rejoining of machine - - if checkCreationTimeout { - // Timeout occurred while machine creation - description = fmt.Sprintf( - "Machine %s failed to join the cluster in %s minutes.", - machine.Name, - timeOutDuration, - ) - } else { - // Timeour occurred due to machine being unhealthy for too long - description = fmt.Sprintf( - "Machine %s is not healthy since %s minutes. Changing status to failed. Node Conditions: %+v", - machine.Name, - timeOutDuration, - machine.Status.Conditions, - ) - } - - lastOperation = v1alpha1.LastOperation{ - Description: description, - State: v1alpha1.MachineStateFailed, - Type: machine.Status.LastOperation.Type, - LastUpdateTime: metav1.Now(), - } - currentStatus = v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineFailed, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - // Log the error message for machine failure - klog.Error(description) - - // Update the machine status to reflect the changes - c.updateMachineStatus(ctx, machine, lastOperation, currentStatus) - - } else { - // If timeout has not occurred, re-enqueue the machine - // after a specified sleep time - c.enqueueMachineAfter(machine, sleepTime) - } - } -} - -func shouldReconcileMachine(machine *v1alpha1.Machine, now time.Time) bool { - if machine.DeletionTimestamp != nil { - return true - } - if machine.Spec.ProviderID == "" { - return true - } - // TODO add more cases where this will be false - - return true -} - -// decodeMachineID is a generic way of decoding the Spec.ProviderID field of node-objects. -func decodeMachineID(id string) string { - splitProviderID := strings.Split(id, "/") - return splitProviderID[len(splitProviderID)-1] -} - -// getEffectiveDrainTimeout returns the drainTimeout set on the machine-object, otherwise returns the timeout set using the global-flag. -func (c *controller) getEffectiveDrainTimeout(machine *v1alpha1.Machine) *metav1.Duration { - var effectiveDrainTimeout *metav1.Duration - if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineConfiguration.MachineDrainTimeout != nil { - effectiveDrainTimeout = machine.Spec.MachineConfiguration.MachineDrainTimeout - } else { - effectiveDrainTimeout = &c.safetyOptions.MachineDrainTimeout - } - return effectiveDrainTimeout -} - -// getEffectiveMaxEvictRetries returns the maxEvictRetries set on the machine-object, otherwise returns the evict retries set using the global-flag. -func (c *controller) getEffectiveMaxEvictRetries(machine *v1alpha1.Machine) *int32 { - var maxEvictRetries *int32 - if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineConfiguration.MaxEvictRetries != nil { - maxEvictRetries = machine.Spec.MachineConfiguration.MaxEvictRetries - } else { - maxEvictRetries = &c.safetyOptions.MaxEvictRetries - } - return maxEvictRetries -} - -// getEffectiveHealthTimeout returns the healthTimeout set on the machine-object, otherwise returns the timeout set using the global-flag. -func (c *controller) getEffectiveHealthTimeout(machine *v1alpha1.Machine) *metav1.Duration { - var effectiveHealthTimeout *metav1.Duration - if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineConfiguration.MachineHealthTimeout != nil { - effectiveHealthTimeout = machine.Spec.MachineConfiguration.MachineHealthTimeout - } else { - effectiveHealthTimeout = &c.safetyOptions.MachineHealthTimeout - } - return effectiveHealthTimeout -} - -// getEffectiveHealthTimeout returns the creationTimeout set on the machine-object, otherwise returns the timeout set using the global-flag. -func (c *controller) getEffectiveCreationTimeout(machine *v1alpha1.Machine) *metav1.Duration { - var effectiveCreationTimeout *metav1.Duration - if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineConfiguration.MachineCreationTimeout != nil { - effectiveCreationTimeout = machine.Spec.MachineConfiguration.MachineCreationTimeout - } else { - effectiveCreationTimeout = &c.safetyOptions.MachineCreationTimeout - } - return effectiveCreationTimeout -} - -// getEffectiveNodeConditions returns the nodeConditions set on the machine-object, otherwise returns the conditions set using the global-flag. -func (c *controller) getEffectiveNodeConditions(machine *v1alpha1.Machine) *string { - var effectiveNodeConditions *string - if machine.Spec.MachineConfiguration != nil && machine.Spec.MachineConfiguration.NodeConditions != nil { - effectiveNodeConditions = machine.Spec.MachineConfiguration.NodeConditions - } else { - effectiveNodeConditions = &c.nodeConditions - } - return effectiveNodeConditions -} diff --git a/pkg/controller/machine_safety.go b/pkg/controller/machine_safety.go index d1852d4cb..9d092d6fa 100644 --- a/pkg/controller/machine_safety.go +++ b/pkg/controller/machine_safety.go @@ -20,10 +20,7 @@ package controller import ( "context" "fmt" - "time" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" intstrutil "k8s.io/apimachinery/pkg/util/intstr" @@ -31,7 +28,6 @@ import ( "github.com/gardener/machine-controller-manager/pkg/util/provider/cache" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/driver" "k8s.io/klog/v2" ) @@ -46,20 +42,6 @@ const ( UnfreezeAnnotation = "safety.machine.sapcloud.io/unfreeze" ) -// reconcileClusterMachineSafetyOrphanVMs checks for any orphan VMs and deletes them -func (c *controller) reconcileClusterMachineSafetyOrphanVMs(key string) error { - ctx := context.Background() - reSyncAfter := c.safetyOptions.MachineSafetyOrphanVMsPeriod.Duration - defer c.machineSafetyOrphanVMsQueue.AddAfter("", reSyncAfter) - - klog.V(4).Infof("reconcileClusterMachineSafetyOrphanVMs: Start") - defer klog.V(4).Infof("reconcileClusterMachineSafetyOrphanVMs: End, reSync-Period: %v", reSyncAfter) - - c.checkVMObjects(ctx) - - return nil -} - // reconcileClusterMachineSafetyOvershooting checks all machineSet/machineDeployment // if the number of machine objects backing them is way beyond its desired replicas func (c *controller) reconcileClusterMachineSafetyOvershooting(key string) error { @@ -99,108 +81,6 @@ func (c *controller) reconcileClusterMachineSafetyOvershooting(key string) error return err } -// reconcileClusterMachineSafetyAPIServer checks control and target clusters -// and checks if their APIServer's are reachable -// If they are not reachable, they set a machineControllerFreeze flag -func (c *controller) reconcileClusterMachineSafetyAPIServer(key string) error { - ctx := context.Background() - statusCheckTimeout := c.safetyOptions.MachineSafetyAPIServerStatusCheckTimeout.Duration - statusCheckPeriod := c.safetyOptions.MachineSafetyAPIServerStatusCheckPeriod.Duration - - klog.V(4).Infof("reconcileClusterMachineSafetyAPIServer: Start") - defer klog.V(4).Infof("reconcileClusterMachineSafetyAPIServer: Stop") - - if c.safetyOptions.MachineControllerFrozen { - // MachineController is frozen - if c.isAPIServerUp(ctx) { - // APIServer is up now, hence we need reset all machine health checks (to avoid unwanted freezes) and unfreeze - machines, err := c.machineLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Unable to LIST machines. Error:", err) - return err - } - for _, machine := range machines { - if machine.Status.CurrentStatus.Phase == v1alpha1.MachineUnknown { - machine, err := c.controlMachineClient.Machines(c.namespace).Get(ctx, machine.Name, metav1.GetOptions{}) - if err != nil { - klog.Error("SafetyController: Unable to GET machines. Error:", err) - return err - } - - machine.Status.CurrentStatus = v1alpha1.CurrentStatus{ - Phase: v1alpha1.MachineRunning, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - } - machine.Status.LastOperation = v1alpha1.LastOperation{ - Description: "Machine Health Timeout was reset due to APIServer being unreachable", - LastUpdateTime: metav1.Now(), - State: v1alpha1.MachineStateSuccessful, - Type: v1alpha1.MachineOperationHealthCheck, - } - _, err = c.controlMachineClient.Machines(c.namespace).UpdateStatus(ctx, machine, metav1.UpdateOptions{}) - if err != nil { - klog.Error("SafetyController: Unable to UPDATE machine/status. Error:", err) - return err - } - - klog.V(2).Info("SafetyController: Reinitializing machine health check for ", machine.Name) - } - - // En-queue after 30 seconds, to ensure all machine states are reconciled - c.enqueueMachineAfter(machine, 30*time.Second) - } - - c.safetyOptions.MachineControllerFrozen = false - c.safetyOptions.APIserverInactiveStartTime = time.Time{} - klog.V(2).Infof("SafetyController: UnFreezing Machine Controller") - } - } else { - // MachineController is not frozen - if !c.isAPIServerUp(ctx) { - // If APIServer is not up - if c.safetyOptions.APIserverInactiveStartTime.Equal(time.Time{}) { - // If timeout has not started - c.safetyOptions.APIserverInactiveStartTime = time.Now() - } - if time.Now().Sub(c.safetyOptions.APIserverInactiveStartTime) > statusCheckTimeout { - // If APIServer has been down for more than statusCheckTimeout - c.safetyOptions.MachineControllerFrozen = true - klog.V(2).Infof("SafetyController: Freezing Machine Controller") - } - - // Re-enqueue the safety check more often if APIServer is not active and is not frozen yet - defer c.machineSafetyAPIServerQueue.AddAfter("", statusCheckTimeout/5) - return nil - } - } - - defer c.machineSafetyAPIServerQueue.AddAfter("", statusCheckPeriod) - return nil -} - -// isAPIServerUp returns true if APIServers are up -// Both control and target APIServers -func (c *controller) isAPIServerUp(ctx context.Context) bool { - // Dummy get call to check if control APIServer is reachable - _, err := c.controlMachineClient.Machines(c.namespace).Get(ctx, "dummy_name", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - // Get returns an error other than object not found = Assume APIServer is not reachable - klog.Error("SafetyController: Unable to GET on machine objects ", err) - return false - } - - // Dummy get call to check if target APIServer is reachable - _, err = c.targetCoreClient.CoreV1().Nodes().Get(ctx, "dummy_name", metav1.GetOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - // Get returns an error other than object not found = Assume APIServer is not reachable - klog.Error("SafetyController: Unable to GET on node objects ", err) - return false - } - - return true -} - // unfreezeMachineDeploymentsWithUnfreezeAnnotation unfreezes machineDeployment with unfreeze annotation func (c *controller) unfreezeMachineDeploymentsWithUnfreezeAnnotation(ctx context.Context) error { machineDeployments, err := c.machineDeploymentLister.List(labels.Everything()) @@ -406,258 +286,6 @@ func (c *controller) checkAndFreezeORUnfreezeMachineSets(ctx context.Context) er return nil } -// checkVMObjects checks for orphan VMs (VMs that don't have a machine object backing) -func (c *controller) checkVMObjects(ctx context.Context) { - c.checkAWSMachineClass(ctx) - c.checkOSMachineClass(ctx) - c.checkAzureMachineClass(ctx) - c.checkGCPMachineClass(ctx) - c.checkAlicloudMachineClass(ctx) - c.checkPacketMachineClass(ctx) -} - -// checkAWSMachineClass checks for orphan VMs in AWSMachinesClasses -func (c *controller) checkAWSMachineClass(ctx context.Context) { - AWSMachineClasses, err := c.awsMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range AWSMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkOSMachineClass checks for orphan VMs in OSMachinesClasses -func (c *controller) checkOSMachineClass(ctx context.Context) { - OSMachineClasses, err := c.openStackMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range OSMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkOSMachineClass checks for orphan VMs in AzureMachinesClasses -func (c *controller) checkAzureMachineClass(ctx context.Context) { - AzureMachineClasses, err := c.azureMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range AzureMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkGCPMachineClass checks for orphan VMs in GCPMachinesClasses -func (c *controller) checkGCPMachineClass(ctx context.Context) { - GCPMachineClasses, err := c.gcpMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range GCPMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkAlicloudMachineClass checks for orphan VMs in AlicloudMachinesClasses -func (c *controller) checkAlicloudMachineClass(ctx context.Context) { - AlicloudMachineClasses, err := c.alicloudMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range AlicloudMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkPacketMachineClass checks for orphan VMs in PacketMachinesClasses -func (c *controller) checkPacketMachineClass(ctx context.Context) { - PacketMachineClasses, err := c.packetMachineClassLister.List(labels.Everything()) - if err != nil { - klog.Error("SafetyController: Error while trying to LIST machineClasses ", err) - return - } - - for _, machineClass := range PacketMachineClasses { - c.checkMachineClass( - ctx, - machineClass, - machineClass.Spec.SecretRef, - machineClass.Spec.CredentialsSecretRef, - machineClass.Name, - machineClass.Kind, - ) - } -} - -// checkMachineClass checks a particular machineClass for orphan instances -func (c *controller) checkMachineClass( - ctx context.Context, - machineClass interface{}, - secretRef *corev1.SecretReference, - credentialsSecretRef *corev1.SecretReference, - className string, - classKind string) { - - // Get secret data - secretData, err := c.getSecretData(className, secretRef, credentialsSecretRef) - if err != nil || secretData == nil { - klog.Errorf("SafetyController: Secret Data could not be computed for MachineClass: %q", className) - return - } - - // Dummy driver object being created to invoke GetVMs - dvr := driver.NewDriver( - "", - secretData, - classKind, - machineClass, - "", - ) - listOfVMs, err := dvr.GetVMs("") - if err != nil { - klog.Errorf("SafetyController: Failed to LIST VMs at provider. Error: %s", err) - } - - // Making sure that its not a VM just being created, machine object not yet updated at API server - if len(listOfVMs) > 1 { - stopCh := make(chan struct{}) - defer close(stopCh) - - if !cache.WaitForCacheSync(stopCh, c.machineSynced) { - klog.Errorf("SafetyController: Timed out waiting for caches to sync. Error: %s", err) - return - } - } - - for machineID, machineName := range listOfVMs { - machine, err := c.machineLister.Machines(c.namespace).Get(machineName) - - if err != nil && !apierrors.IsNotFound(err) { - // Any other types of errors - klog.Errorf("SafetyController: Error while trying to GET machines. Error: %s", err) - } else if err != nil || machine.Spec.ProviderID != machineID { - // If machine exists and machine object is still been processed by the machine controller - if err == nil && - (machine.Status.CurrentStatus.Phase == "" || machine.Status.CurrentStatus.Phase == v1alpha1.MachineCrashLoopBackOff) { - klog.V(3).Infof("SafetyController: Machine object %q is being processed by machine controller, hence skipping", machine.Name) - continue - } - - // Re-check VM object existence - // before deleting orphan VM - result, _ := dvr.GetVMs(machineID) - for reMachineID := range result { - if reMachineID == machineID { - // Get latest version of machine object and verfiy again - machine, err := c.controlMachineClient.Machines(c.namespace).Get(ctx, machineName, metav1.GetOptions{}) - if (err != nil && apierrors.IsNotFound(err)) || machine.Spec.ProviderID != machineID { - vm := make(map[string]string) - vm[machineID] = machineName - c.deleteOrphanVM(vm, secretData, classKind, machineClass) - } - } - } - - } - } -} - -// addMachineToSafety enqueues into machineSafetyQueue when a new machine is added -func (c *controller) addMachineToSafety(obj interface{}) { - machine := obj.(*v1alpha1.Machine) - c.enqueueMachineSafetyOvershootingKey(machine) -} - -// deleteMachineToSafety enqueues into machineSafetyQueue when a new machine is deleted -func (c *controller) deleteMachineToSafety(obj interface{}) { - machine := obj.(*v1alpha1.Machine) - c.enqueueMachineSafetyOrphanVMsKey(machine) -} - -// enqueueMachineSafetyOvershootingKey enqueues into machineSafetyOvershootingQueue -func (c *controller) enqueueMachineSafetyOvershootingKey(obj interface{}) { - c.machineSafetyOvershootingQueue.Add("") -} - -// enqueueMachineSafetyOrphanVMsKey enqueues into machineSafetyOrphanVMsQueue -func (c *controller) enqueueMachineSafetyOrphanVMsKey(obj interface{}) { - c.machineSafetyOrphanVMsQueue.Add("") -} - -// deleteOrphanVM teriminates's the VM on the cloud provider passed to it -func (c *controller) deleteOrphanVM(vm driver.VMs, secretData map[string][]byte, kind string, machineClass interface{}) { - - var machineID string - var machineName string - - for k, v := range vm { - machineID = k - machineName = v - } - - dvr := driver.NewDriver( - machineID, - secretData, - kind, - machineClass, - machineName, - ) - - err := dvr.Delete(machineID) - if err != nil { - klog.Errorf("SafetyController: Error while trying to DELETE VM on CP - %s. Shall retry in next safety controller sync.", err) - } else { - klog.V(2).Infof("SafetyController: Orphan VM found and terminated VM: %s, %s", machineName, machineID) - } -} - // freezeMachineSetAndDeployment freezes machineSet and machineDeployment (who is the owner of the machineSet) func (c *controller) freezeMachineSetAndDeployment(ctx context.Context, machineSet *v1alpha1.MachineSet, reason string, message string) error { diff --git a/pkg/controller/machine_safety_test.go b/pkg/controller/machine_safety_test.go index b4437eae6..a61ead1ea 100644 --- a/pkg/controller/machine_safety_test.go +++ b/pkg/controller/machine_safety_test.go @@ -17,26 +17,24 @@ package controller import ( "context" - "time" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" machinev1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/driver" cacheutil "github.com/gardener/machine-controller-manager/pkg/util/provider/cache" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" ) +const testNamespace = "test" + var _ = Describe("#machine_safety", func() { DescribeTable("##checkAndFreezeORUnfreezeMachineSets", func( - machineSet *v1alpha1.MachineSet, + machineSet *machinev1.MachineSet, machineCount int, FreezeMachineSet bool, ) { @@ -71,7 +69,7 @@ var _ = Describe("#machine_safety", func() { // When there is 1 machineSet with 1 replica and corresponding 1 machine Object // MachineSet is not frozen - Entry("MachineSet with 1 replica & 1 machine object", newMachineSet(&v1alpha1.MachineTemplateSpec{ + Entry("MachineSet with 1 replica & 1 machine object", newMachineSet(&machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -80,7 +78,7 @@ var _ = Describe("#machine_safety", func() { // When there is 1 machineSet with 1 replica and corresponding 5 machine Object // MachineSet is frozen by adding a label on MachineSet - Entry("MachineSet with 1 replica & 5 machine object", newMachineSet(&v1alpha1.MachineTemplateSpec{ + Entry("MachineSet with 1 replica & 5 machine object", newMachineSet(&machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -90,7 +88,7 @@ var _ = Describe("#machine_safety", func() { // When there is 1 machineSet with 5 replica and corresponding 2 machine Object // with a {"Freeze": "True"} label set on the MachineSe, the MachineSet is unfrozen // by removing a label on MachineSet - Entry("MachineSet with 5 replica & 2 machine object", newMachineSet(&v1alpha1.MachineTemplateSpec{ + Entry("MachineSet with 5 replica & 2 machine object", newMachineSet(&machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -99,7 +97,7 @@ var _ = Describe("#machine_safety", func() { ) DescribeTable("##freezeMachineSetsAndDeployments", - func(machineSet *v1alpha1.MachineSet) { + func(machineSet *machinev1.MachineSet) { stop := make(chan struct{}) defer close(stop) @@ -127,7 +125,7 @@ var _ = Describe("#machine_safety", func() { Expect(ms.Labels["freeze"]).To(Equal("True")) } }, - Entry("one machineset", newMachineSet(&v1alpha1.MachineTemplateSpec{ + Entry("one machineset", newMachineSet(&machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -137,7 +135,7 @@ var _ = Describe("#machine_safety", func() { DescribeTable("##unfreezeMachineSetsAndDeployments", func(machineSetExists, machineSetIsFrozen, parentExists, parentIsFrozen bool) { - testMachineSet := newMachineSet(&v1alpha1.MachineTemplateSpec{ + testMachineSet := newMachineSet(&machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -149,17 +147,17 @@ var _ = Describe("#machine_safety", func() { if machineSetIsFrozen { testMachineSet.Labels["freeze"] = "True" msStatus := testMachineSet.Status - mscond := NewMachineSetCondition(v1alpha1.MachineSetFrozen, v1alpha1.ConditionTrue, "testing", "freezing the machineset") + mscond := NewMachineSetCondition(machinev1.MachineSetFrozen, machinev1.ConditionTrue, "testing", "freezing the machineset") SetCondition(&msStatus, mscond) } - testMachineDeployment := &v1alpha1.MachineDeployment{ + testMachineDeployment := &machinev1.MachineDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: "testMachineDeployment", Namespace: testNamespace, Labels: map[string]string{}, }, - Spec: v1alpha1.MachineDeploymentSpec{ + Spec: machinev1.MachineDeploymentSpec{ Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "name": "testMachineDeployment", @@ -170,7 +168,7 @@ var _ = Describe("#machine_safety", func() { if parentIsFrozen { testMachineDeployment.Labels["freeze"] = "True" mdStatus := testMachineDeployment.Status - mdCond := NewMachineDeploymentCondition(v1alpha1.MachineDeploymentFrozen, v1alpha1.ConditionTrue, "testing", "freezing the machinedeployment") + mdCond := NewMachineDeploymentCondition(machinev1.MachineDeploymentFrozen, machinev1.ConditionTrue, "testing", "freezing the machinedeployment") SetMachineDeploymentCondition(&mdStatus, *mdCond) } @@ -193,11 +191,11 @@ var _ = Describe("#machine_safety", func() { machineSet, err := c.controlMachineClient.MachineSets(testMachineSet.Namespace).Get(context.TODO(), testMachineSet.Name, metav1.GetOptions{}) if machineSetExists { Expect(machineSet.Labels["freeze"]).Should((BeEmpty())) - Expect(GetCondition(&machineSet.Status, v1alpha1.MachineSetFrozen)).Should(BeNil()) + Expect(GetCondition(&machineSet.Status, machinev1.MachineSetFrozen)).Should(BeNil()) machineDeployment, err := c.controlMachineClient.MachineDeployments(testMachineDeployment.Namespace).Get(context.TODO(), testMachineDeployment.Name, metav1.GetOptions{}) if parentExists { //Expect(machineDeployment.Labels["freeze"]).Should((BeEmpty())) - Expect(GetMachineDeploymentCondition(machineDeployment.Status, v1alpha1.MachineDeploymentFrozen)).Should(BeNil()) + Expect(GetMachineDeploymentCondition(machineDeployment.Status, machinev1.MachineDeploymentFrozen)).Should(BeNil()) } else { Expect(err).ShouldNot(BeNil()) } @@ -216,24 +214,21 @@ var _ = Describe("#machine_safety", func() { machineReplicas int machineSetAnnotations map[string]string machineSetLabels map[string]string - machineSetConditions []v1alpha1.MachineSetCondition + machineSetConditions []machinev1.MachineSetCondition machineDeploymentAnnotations map[string]string machineDeploymentLabels map[string]string - machineDeploymentConditions []v1alpha1.MachineDeploymentCondition - } - type action struct { + machineDeploymentConditions []machinev1.MachineDeploymentCondition } type expect struct { machineSetAnnotations map[string]string machineSetLabels map[string]string - machineSetConditions []v1alpha1.MachineSetCondition + machineSetConditions []machinev1.MachineSetCondition machineDeploymentAnnotations map[string]string machineDeploymentLabels map[string]string - machineDeploymentConditions []v1alpha1.MachineDeploymentCondition + machineDeploymentConditions []machinev1.MachineDeploymentCondition } type data struct { setup setup - action action expect expect } @@ -243,7 +238,7 @@ var _ = Describe("#machine_safety", func() { defer close(stop) machineDeployment := newMachineDeployment( - &v1alpha1.MachineTemplateSpec{ + &machinev1.MachineTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: "machine", Namespace: testNamespace, @@ -254,7 +249,7 @@ var _ = Describe("#machine_safety", func() { }, 1, 10, - &v1alpha1.MachineDeploymentStatus{ + &machinev1.MachineDeploymentStatus{ Conditions: data.setup.machineDeploymentConditions, }, nil, @@ -264,7 +259,7 @@ var _ = Describe("#machine_safety", func() { machineSet := newMachineSetFromMachineDeployment( machineDeployment, 1, - &v1alpha1.MachineSetStatus{ + &machinev1.MachineSetStatus{ Conditions: data.setup.machineSetConditions, }, data.setup.machineSetAnnotations, @@ -273,7 +268,7 @@ var _ = Describe("#machine_safety", func() { machines := newMachinesFromMachineSet( data.setup.machineReplicas, machineSet, - &v1alpha1.MachineStatus{}, + &machinev1.MachineStatus{}, nil, nil, ) @@ -358,20 +353,20 @@ var _ = Describe("#machine_safety", func() { "freeze": "True", "machine": "test", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -383,20 +378,20 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -414,20 +409,20 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -437,20 +432,20 @@ var _ = Describe("#machine_safety", func() { "freeze": "True", "machine": "test", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -459,10 +454,10 @@ var _ = Describe("#machine_safety", func() { Entry("Unfreezing machineset with freeze conditions and none on machinedeployment", &data{ setup: setup{ machineReplicas: 3, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -491,17 +486,17 @@ var _ = Describe("#machine_safety", func() { Entry("Unfreezing machineset with freeze label and freeze label on machinedeployment", &data{ setup: setup{ machineReplicas: 3, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -541,20 +536,20 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -567,10 +562,10 @@ var _ = Describe("#machine_safety", func() { machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -582,10 +577,10 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -595,10 +590,10 @@ var _ = Describe("#machine_safety", func() { machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -622,10 +617,10 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -635,10 +630,10 @@ var _ = Describe("#machine_safety", func() { machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -655,10 +650,10 @@ var _ = Describe("#machine_safety", func() { Entry("Unfreeze a machineDeployment with freeze condition and no machinesets frozen", &data{ setup: setup{ machineReplicas: 1, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -690,17 +685,17 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -710,20 +705,20 @@ var _ = Describe("#machine_safety", func() { "machine": "test", "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -735,10 +730,10 @@ var _ = Describe("#machine_safety", func() { machineSetLabels: map[string]string{ "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -748,20 +743,20 @@ var _ = Describe("#machine_safety", func() { "machine": "test", "freeze": "True", }, - machineSetConditions: []v1alpha1.MachineSetCondition{ + machineSetConditions: []machinev1.MachineSetCondition{ { - Type: v1alpha1.MachineSetFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineSetFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, machineDeploymentLabels: map[string]string{ "freeze": "True", }, - machineDeploymentConditions: []v1alpha1.MachineDeploymentCondition{ + machineDeploymentConditions: []machinev1.MachineDeploymentCondition{ { - Type: v1alpha1.MachineDeploymentFrozen, - Status: v1alpha1.ConditionTrue, + Type: machinev1.MachineDeploymentFrozen, + Status: machinev1.ConditionTrue, Message: "The number of machines backing MachineSet: machineset-0 is 4 >= 4 which is the Max-ScaleUp-Limit", }, }, @@ -769,216 +764,216 @@ var _ = Describe("#machine_safety", func() { }), ) - const ( - zeroDuration = time.Duration(0) - fiveSecondsDuration = 5 * time.Second - fiveMinutesDuration = 5 * time.Minute - ) - DescribeTable("##reconcileClusterMachineSafetyAPIServer", - func( - controlAPIServerIsUp bool, - targetAPIServerIsUp bool, - apiServerInactiveDuration time.Duration, - preMachineControllerIsFrozen bool, - postMachineControllerFrozen bool, - ) { - apiServerInactiveStartTime := time.Now().Add(-apiServerInactiveDuration) - stop := make(chan struct{}) - defer close(stop) - - testMachine := &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine1", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineUnknown, - }, - }, - } - controlMachineObjects := []runtime.Object{} - controlMachineObjects = append(controlMachineObjects, testMachine) - - c, trackers := createController(stop, testNamespace, controlMachineObjects, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - c.safetyOptions.APIserverInactiveStartTime = apiServerInactiveStartTime - c.safetyOptions.MachineControllerFrozen = preMachineControllerIsFrozen - if !controlAPIServerIsUp { - trackers.ControlMachine.SetError("APIServer is Not Reachable") - trackers.ControlCore.SetError("APIServer is Not Reachable") - } - if !targetAPIServerIsUp { - trackers.TargetCore.SetError("APIServer is Not Reachable") - } - - c.reconcileClusterMachineSafetyAPIServer("") - - Expect(c.safetyOptions.MachineControllerFrozen).Should(Equal(postMachineControllerFrozen)) - }, - - // Both APIServers are reachable - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", - true, true, zeroDuration, false, false), - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: false", - true, true, zeroDuration, true, false), - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", - true, true, fiveSecondsDuration, false, false), - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: false", - true, true, fiveSecondsDuration, true, false), - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: false", - true, true, fiveMinutesDuration, false, false), - Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: false", - true, true, fiveMinutesDuration, true, false), - - // Target APIServer is not reachable - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", - true, false, zeroDuration, false, false), - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", - true, false, zeroDuration, true, true), - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", - true, false, fiveSecondsDuration, false, false), - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", - true, false, fiveSecondsDuration, true, true), - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", - true, false, fiveMinutesDuration, false, true), - Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", - true, false, fiveMinutesDuration, true, true), - - // Control APIServer is not reachable - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", - false, true, zeroDuration, false, false), - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", - false, true, zeroDuration, true, true), - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", - false, true, fiveSecondsDuration, false, false), - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", - false, true, fiveSecondsDuration, true, true), - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", - false, true, fiveMinutesDuration, false, true), - Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", - false, true, fiveMinutesDuration, true, true), - - // Both APIServers are not reachable - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", - false, false, zeroDuration, false, false), - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", - false, false, zeroDuration, true, true), - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", - false, false, fiveSecondsDuration, false, false), - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", - false, false, fiveSecondsDuration, true, true), - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", - false, false, fiveMinutesDuration, false, true), - Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", - false, false, fiveMinutesDuration, true, true), - ) + // const ( + // zeroDuration = time.Duration(0) + // fiveSecondsDuration = 5 * time.Second + // fiveMinutesDuration = 5 * time.Minute + // ) + // DescribeTable("##reconcileClusterMachineSafetyAPIServer", + // func( + // controlAPIServerIsUp bool, + // targetAPIServerIsUp bool, + // apiServerInactiveDuration time.Duration, + // preMachineControllerIsFrozen bool, + // postMachineControllerFrozen bool, + // ) { + // apiServerInactiveStartTime := time.Now().Add(-apiServerInactiveDuration) + // stop := make(chan struct{}) + // defer close(stop) + + // testMachine := &machinev1.Machine{ + // ObjectMeta: metav1.ObjectMeta{ + // Name: "testmachine1", + // Namespace: testNamespace, + // }, + // Status: machinev1.MachineStatus{ + // CurrentStatus: machinev1.CurrentStatus{ + // Phase: machinev1.MachineUnknown, + // }, + // }, + // } + // controlMachineObjects := []runtime.Object{} + // controlMachineObjects = append(controlMachineObjects, testMachine) + + // c, trackers := createController(stop, testNamespace, controlMachineObjects, nil, nil) + // defer trackers.Stop() + // waitForCacheSync(stop, c) + + // c.safetyOptions.APIserverInactiveStartTime = apiServerInactiveStartTime + // c.safetyOptions.MachineControllerFrozen = preMachineControllerIsFrozen + // if !controlAPIServerIsUp { + // trackers.ControlMachine.SetError("APIServer is Not Reachable") + // trackers.ControlCore.SetError("APIServer is Not Reachable") + // } + // if !targetAPIServerIsUp { + // trackers.TargetCore.SetError("APIServer is Not Reachable") + // } + + // c.reconcileClusterMachineSafetyAPIServer("") + + // Expect(c.safetyOptions.MachineControllerFrozen).Should(Equal(postMachineControllerFrozen)) + // }, + + // // Both APIServers are reachable + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", + // true, true, zeroDuration, false, false), + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: false", + // true, true, zeroDuration, true, false), + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", + // true, true, fiveSecondsDuration, false, false), + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: false", + // true, true, fiveSecondsDuration, true, false), + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: false", + // true, true, fiveMinutesDuration, false, false), + // Entry("Control APIServer: Reachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: false", + // true, true, fiveMinutesDuration, true, false), + + // // Target APIServer is not reachable + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", + // true, false, zeroDuration, false, false), + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", + // true, false, zeroDuration, true, true), + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", + // true, false, fiveSecondsDuration, false, false), + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", + // true, false, fiveSecondsDuration, true, true), + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", + // true, false, fiveMinutesDuration, false, true), + // Entry("Control APIServer: Reachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", + // true, false, fiveMinutesDuration, true, true), + + // // Control APIServer is not reachable + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", + // false, true, zeroDuration, false, false), + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", + // false, true, zeroDuration, true, true), + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", + // false, true, fiveSecondsDuration, false, false), + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", + // false, true, fiveSecondsDuration, true, true), + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", + // false, true, fiveMinutesDuration, false, true), + // Entry("Control APIServer: UnReachable, Target APIServer: Reachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", + // false, true, fiveMinutesDuration, true, true), + + // // Both APIServers are not reachable + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: false = Post-Frozen: false", + // false, false, zeroDuration, false, false), + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Inactive, Pre-Frozen: true = Post-Frozen: true", + // false, false, zeroDuration, true, true), + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: false = Post-Frozen: false", + // false, false, fiveSecondsDuration, false, false), + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Started, Pre-Frozen: true = Post-Frozen: true", + // false, false, fiveSecondsDuration, true, true), + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: false = Post-Frozen: true", + // false, false, fiveMinutesDuration, false, true), + // Entry("Control APIServer: UnReachable, Target APIServer: UnReachable, Inactive Timer: Elapsed, Pre-Frozen: true = Post-Frozen: true", + // false, false, fiveMinutesDuration, true, true), + // ) }) -var _ = Describe("machineCrashloopBackoff", func() { - objMeta := &metav1.ObjectMeta{ - GenerateName: "class", - Namespace: testNamespace, - } - - classKind := "MachineClass" - secretData := map[string][]byte{ - "userData": []byte("dummy-data"), - "azureClientId": []byte("dummy-client-id"), - "azureClientSecret": []byte("dummy-client-secret"), - "azureSubscriptionId": []byte("dummy-subcription-id"), - "azureTenantId": []byte("dummy-tenant-id"), - } - - Describe("machineCrashloopBackoff", func() { - - It("Should delete the machine (old code)", func() { - stop := make(chan struct{}) - defer close(stop) - - // Create test secret and add it to controlCoreObject list - testSecret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: testNamespace, - }, - Data: secretData, - } - - // Create a test secretReference because the method checkMachineClass needs it - testSecretReference := &v1.SecretReference{ - Name: "test-secret", - Namespace: testNamespace, - } - - testMachineClass := &machinev1.MachineClass{ - ObjectMeta: *newObjectMeta(objMeta, 0), - SecretRef: testSecretReference, - } - controlCoreObjects := []runtime.Object{} - controlCoreObjects = append(controlCoreObjects, testSecret) - - // Create test machine object in CrashloopBackoff state - testMachineObject1 := &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine_1", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineCrashLoopBackOff, - }, - }, - } - - // Create another test machine object in Running state - testMachineObject2 := &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine_2", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineRunning, - }, - }, - } - - controlMachineObjects := []runtime.Object{} - controlMachineObjects = append(controlMachineObjects, testMachineObject1) - controlMachineObjects = append(controlMachineObjects, testMachineObject2) - - c, trackers := createController(stop, testNamespace, controlMachineObjects, controlCoreObjects, nil) - defer trackers.Stop() - - fakeDriver := driver.NewDriver( - "", - secretData, - classKind, - testMachineClass, - "", - ) - - // use type assertion to let Golang know that the - // returned fakeDriver is of type *driver.FakeDriver - fd := fakeDriver.(*driver.FakeDriver) - - _ = fd.Add("testmachine-ip1", "testmachine_1") - _ = fd.Add("testmachine-ip2", "testmachine_2") - - waitForCacheSync(stop, c) - - // call checkMachineClass to delete the orphan VMs - c.checkMachineClass(context.TODO(), testMachineClass, testSecretReference, testSecretReference, objMeta.Name, classKind) - - listOfVMs, _ := fakeDriver.GetVMs("") - - // after this, the testmachine in crashloopbackoff phase should remain and the other one should - // be deleted because it is an orphan VM - Expect(listOfVMs["testmachine-ip1"]).To(Equal("testmachine_1")) - Expect(listOfVMs["testmachine-ip2"]).To(Equal("")) - }) - }) -}) +// var _ = Describe("machineCrashloopBackoff", func() { +// objMeta := &metav1.ObjectMeta{ +// GenerateName: "class", +// Namespace: testNamespace, +// } + +// classKind := "MachineClass" +// secretData := map[string][]byte{ +// "userData": []byte("dummy-data"), +// "azureClientId": []byte("dummy-client-id"), +// "azureClientSecret": []byte("dummy-client-secret"), +// "azureSubscriptionId": []byte("dummy-subcription-id"), +// "azureTenantId": []byte("dummy-tenant-id"), +// } + +// Describe("machineCrashloopBackoff", func() { + +// It("Should delete the machine (old code)", func() { +// stop := make(chan struct{}) +// defer close(stop) + +// // Create test secret and add it to controlCoreObject list +// testSecret := &v1.Secret{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "test-secret", +// Namespace: testNamespace, +// }, +// Data: secretData, +// } + +// // Create a test secretReference because the method checkMachineClass needs it +// testSecretReference := &v1.SecretReference{ +// Name: "test-secret", +// Namespace: testNamespace, +// } + +// testMachineClass := &machinev1.MachineClass{ +// ObjectMeta: *newObjectMeta(objMeta, 0), +// SecretRef: testSecretReference, +// } +// controlCoreObjects := []runtime.Object{} +// controlCoreObjects = append(controlCoreObjects, testSecret) + +// // Create test machine object in CrashloopBackoff state +// testMachineObject1 := &machinev1.Machine{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "testmachine_1", +// Namespace: testNamespace, +// }, +// Status: machinev1.MachineStatus{ +// CurrentStatus: machinev1.CurrentStatus{ +// Phase: machinev1.MachineCrashLoopBackOff, +// }, +// }, +// } + +// // Create another test machine object in Running state +// testMachineObject2 := &machinev1.Machine{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "testmachine_2", +// Namespace: testNamespace, +// }, +// Status: machinev1.MachineStatus{ +// CurrentStatus: machinev1.CurrentStatus{ +// Phase: machinev1.MachineRunning, +// }, +// }, +// } + +// controlMachineObjects := []runtime.Object{} +// controlMachineObjects = append(controlMachineObjects, testMachineObject1) +// controlMachineObjects = append(controlMachineObjects, testMachineObject2) + +// c, trackers := createController(stop, testNamespace, controlMachineObjects, controlCoreObjects, nil) +// defer trackers.Stop() + +// fakeDriver := driver.NewDriver( +// "", +// secretData, +// classKind, +// testMachineClass, +// "", +// ) + +// // use type assertion to let Golang know that the +// // returned fakeDriver is of type *driver.FakeDriver +// fd := fakeDriver.(*driver.FakeDriver) + +// _ = fd.Add("testmachine-ip1", "testmachine_1") +// _ = fd.Add("testmachine-ip2", "testmachine_2") + +// waitForCacheSync(stop, c) + +// // call checkMachineClass to delete the orphan VMs +// c.checkMachineClass(context.TODO(), testMachineClass, testSecretReference, testSecretReference, objMeta.Name, classKind) + +// listOfVMs, _ := fakeDriver.GetVMs("") + +// // after this, the testmachine in crashloopbackoff phase should remain and the other one should +// // be deleted because it is an orphan VM +// Expect(listOfVMs["testmachine-ip1"]).To(Equal("testmachine_1")) +// Expect(listOfVMs["testmachine-ip2"]).To(Equal("")) +// }) +// }) +// }) diff --git a/pkg/controller/machine_test.go b/pkg/controller/machine_test.go deleted file mode 100644 index a3761519b..000000000 --- a/pkg/controller/machine_test.go +++ /dev/null @@ -1,2200 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package controller - -import ( - "context" - "errors" - "fmt" - "math" - "net/http" - "time" - - machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - machinev1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - fakemachineapi "github.com/gardener/machine-controller-manager/pkg/client/clientset/versioned/typed/machine/v1alpha1/fake" - "github.com/gardener/machine-controller-manager/pkg/driver" - customfake "github.com/gardener/machine-controller-manager/pkg/fakeclient" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/validation/field" - k8stesting "k8s.io/client-go/testing" - "k8s.io/klog/v2" -) - -const testNamespace = "test" - -var _ = Describe("machine", func() { - var ( - fakeMachineClient *fakemachineapi.FakeMachineV1alpha1 - c *controller - ) - - Describe("getSecret", func() { - - var ( - secretRef *v1.SecretReference - ) - - BeforeEach(func() { - - secretRef = &v1.SecretReference{ - Name: "test-secret", - Namespace: testNamespace, - } - }) - - It("should return success", func() { - stop := make(chan struct{}) - defer close(stop) - - testSecret := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-secret", - Namespace: testNamespace, - }, - } - objects := []runtime.Object{} - objects = append(objects, testSecret) - - c, trackers := createController(stop, testNamespace, nil, objects, nil) - defer trackers.Stop() - - waitForCacheSync(stop, c) - secretRet, errRet := c.getSecret(secretRef, "test-aws") - Expect(errRet).Should(BeNil()) - Expect(secretRet).Should(Not(BeNil())) - Expect(secretRet).Should(Equal(testSecret)) - - }) - - It("should return error", func() { - err := errors.New("secret \"test-secret\" not found") - stop := make(chan struct{}) - defer close(stop) - - c, trackers := createController(stop, testNamespace, nil, nil, nil) - defer trackers.Stop() - - waitForCacheSync(stop, c) - secretRet, errRet := c.getSecret(secretRef, "test-aws") - Expect(errRet).Should(Not(BeNil())) - Expect(errRet.Error()).Should(Equal(err.Error())) - Expect(secretRet).Should(BeNil()) - }) - - }) - - Describe("#updateMachineStatus", func() { - var ( - machine *machinev1.Machine - lastOperation machinev1.LastOperation - currentStatus machinev1.CurrentStatus - ) - - BeforeEach(func() { - fakeMachineClient = &fakemachineapi.FakeMachineV1alpha1{ - Fake: &k8stesting.Fake{}, - } - c = &controller{ - controlMachineClient: fakeMachineClient, - } - machine = &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-1", - Namespace: testNamespace, - }, - } - lastOperation = machinev1.LastOperation{ - Description: "test operation", - LastUpdateTime: metav1.Now(), - State: machinev1.MachineStateProcessing, - Type: "Create", - } - currentStatus = machinev1.CurrentStatus{ - LastUpdateTime: lastOperation.LastUpdateTime, - Phase: machinev1.MachinePending, - TimeoutActive: true, - } - }) - - It("should return error", func() { - err := errors.New("test error") - - fakeMachineClient.AddReactor("get", "machines", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, nil, err - }) - - machineRet, errRet := c.updateMachineStatus(context.TODO(), machine, lastOperation, currentStatus) - Expect(errRet).Should(Not(BeNil())) - Expect(errRet).Should(BeIdenticalTo(err)) - Expect(machineRet).Should(BeNil()) - }) - - It("shouldn't update status when it is already same", func() { - machine.Status.LastOperation = lastOperation - machine.Status.CurrentStatus = currentStatus - - lastOperation = machinev1.LastOperation{ - Description: "test operation dummy", - LastUpdateTime: metav1.Now(), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - } - currentStatus = machinev1.CurrentStatus{ - LastUpdateTime: lastOperation.LastUpdateTime, - Phase: machinev1.MachinePending, - TimeoutActive: true, - } - - fakeMachineClient.AddReactor("get", "machines", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, machine, nil - }) - - machineRet, errRet := c.updateMachineStatus(context.TODO(), machine, lastOperation, currentStatus) - Expect(errRet).Should((BeNil())) - Expect(machineRet.Status.LastOperation).Should(BeIdenticalTo(machine.Status.LastOperation)) - Expect(machineRet.Status.CurrentStatus).Should(BeIdenticalTo(machine.Status.CurrentStatus)) - }) - - It("should return success", func() { - fakeMachineClient.AddReactor("get", "machines", func(action k8stesting.Action) (bool, runtime.Object, error) { - if action.(k8stesting.GetAction).GetName() == machine.GetName() { - return true, machine, nil - } - return false, nil, nil - }) - - var machineUpdated *machinev1.Machine - fakeMachineClient.AddReactor("update", "machines", func(action k8stesting.Action) (bool, runtime.Object, error) { - o := action.(k8stesting.UpdateAction).GetObject() - if o == nil { - return false, nil, nil - } - - m := o.(*machinev1.Machine) - if m.GetName() == machine.GetName() { - machineUpdated = m - return true, m, nil - } - - return false, nil, nil - - }) - - machineRet, errRet := c.updateMachineStatus(context.TODO(), machine, lastOperation, currentStatus) - Expect(errRet).Should(BeNil()) - Expect(machineRet).Should(Not(BeNil())) - Expect(machineUpdated).Should(Not(BeNil())) - - Expect(machineUpdated).Should(Not(BeIdenticalTo(machine))) - Expect(machineRet).Should(Not(BeIdenticalTo(machine))) - Expect(machineRet).Should(BeIdenticalTo(machineUpdated)) - Expect(machineRet.Status.CurrentStatus).Should(BeIdenticalTo(currentStatus)) - Expect(machineRet.Status.LastOperation).Should(BeIdenticalTo(lastOperation)) - }) - }) - - Describe("#isHealthy", func() { - BeforeEach(func() { - fakeMachineClient = &fakemachineapi.FakeMachineV1alpha1{ - Fake: &k8stesting.Fake{}, - } - c = &controller{ - controlMachineClient: fakeMachineClient, - nodeConditions: "ReadonlyFilesystem,KernelDeadlock,DiskPressure,NetworkUnavailable", - } - }) - - testMachine := machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - Conditions: []corev1.NodeCondition{}, - }, - } - DescribeTable("Checking health of the machine", - func(conditionType corev1.NodeConditionType, conditionStatus corev1.ConditionStatus, expected bool) { - testMachine.Status.Conditions = []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - { - Type: corev1.NodeDiskPressure, - Status: corev1.ConditionFalse, - }, - { - Type: corev1.NodeMemoryPressure, - Status: corev1.ConditionFalse, - }, - { - Type: corev1.NodeNetworkUnavailable, - Status: corev1.ConditionFalse, - }, - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - } - for i, condition := range testMachine.Status.Conditions { - if condition.Type == conditionType { - testMachine.Status.Conditions[i].Status = conditionStatus - break - } - } - Expect(c.isHealthy(&testMachine)).Should(BeIdenticalTo(expected)) - }, - Entry("with NodeReady is True", corev1.NodeReady, corev1.ConditionTrue, true), - Entry("with NodeReady is False", corev1.NodeReady, corev1.ConditionFalse, false), - Entry("with NodeReady is Unknown", corev1.NodeReady, corev1.ConditionUnknown, false), - - Entry("with NodeDiskPressure is True", corev1.NodeDiskPressure, corev1.ConditionTrue, false), - Entry("with NodeDiskPressure is False", corev1.NodeDiskPressure, corev1.ConditionFalse, true), - Entry("with NodeDiskPressure is Unknown", corev1.NodeDiskPressure, corev1.ConditionUnknown, false), - - Entry("with NodeMemoryPressure is True", corev1.NodeMemoryPressure, corev1.ConditionTrue, true), - Entry("with NodeMemoryPressure is False", corev1.NodeMemoryPressure, corev1.ConditionFalse, true), - Entry("with NodeMemoryPressure is Unknown", corev1.NodeMemoryPressure, corev1.ConditionUnknown, true), - - Entry("with NodeNetworkUnavailable is True", corev1.NodeNetworkUnavailable, corev1.ConditionTrue, false), - Entry("with NodeNetworkUnavailable is False", corev1.NodeNetworkUnavailable, corev1.ConditionFalse, true), - Entry("with NodeNetworkUnavailable is Unknown", corev1.NodeNetworkUnavailable, corev1.ConditionUnknown, false), - - Entry("with NodeReady is True", corev1.NodeReady, corev1.ConditionTrue, true), - Entry("with NodeReady is False", corev1.NodeReady, corev1.ConditionFalse, false), - Entry("with NodeReady is Unknown", corev1.NodeReady, corev1.ConditionUnknown, false), - ) - }) - - Describe("##updateMachineConditions", func() { - Describe("Update conditions of a non-existing machine", func() { - It("should return error", func() { - stop := make(chan struct{}) - defer close(stop) - - objects := []runtime.Object{} - c, trackers := createController(stop, testNamespace, objects, nil, nil) - defer trackers.Stop() - - testMachine := &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineTerminating, - }, - }, - } - conditions := []corev1.NodeCondition{} - var _, err = c.updateMachineConditions(context.TODO(), testMachine, conditions) - Expect(err).Should(Not(BeNil())) - }) - }) - DescribeTable("Update conditions of an existing machine", - func(phase machinev1.MachinePhase, conditions []corev1.NodeCondition, expectedPhase machinev1.MachinePhase) { - stop := make(chan struct{}) - defer close(stop) - - testMachine := &machinev1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: "testmachine", - Namespace: testNamespace, - }, - Status: machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: phase, - }, - }, - } - objects := []runtime.Object{} - objects = append(objects, testMachine) - - c, trackers := createController(stop, testNamespace, objects, nil, nil) - defer trackers.Stop() - - var updatedMachine, err = c.updateMachineConditions(context.TODO(), testMachine, conditions) - Expect(updatedMachine.Status.Conditions).Should(BeEquivalentTo(conditions)) - Expect(updatedMachine.Status.CurrentStatus.Phase).Should(BeIdenticalTo(expectedPhase)) - Expect(err).Should(BeNil()) - }, - Entry("healthy status but machine terminating", machinev1.MachineTerminating, []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - }, machinev1.MachineTerminating), - Entry("unhealthy status but machine running", machinev1.MachineRunning, []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionFalse, - }, - }, machinev1.MachineUnknown), - Entry("healthy status but machine not running", machinev1.MachineAvailable, []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, - }, - }, machinev1.MachineRunning), - ) - }) - - Describe("#ValidateMachine", func() { - type data struct { - action machineapi.Machine - expect field.ErrorList - } - DescribeTable("#happy path", - func(data *data) { - errList := validation.ValidateMachine(&data.action) - Expect(errList).To(Equal(data.expect)) - }, - Entry("aws", &data{ - action: machineapi.Machine{ - Spec: machineapi.MachineSpec{ - Class: machineapi.ClassSpec{ - Kind: "AWSMachineClass", - Name: "aws", - }, - }, - }, - expect: field.ErrorList{}, - }), - ) - }) - - Describe("#isMachineStatusEqual", func() { - type expect struct { - equal bool - } - type action struct { - s1 machinev1.MachineStatus - s2 machinev1.MachineStatus - } - type data struct { - action action - expect expect - } - - lastOperation := machinev1.LastOperation{ - Description: "test operation", - LastUpdateTime: metav1.Now(), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - } - currentStatus := machinev1.CurrentStatus{ - LastUpdateTime: lastOperation.LastUpdateTime, - Phase: machinev1.MachinePending, - TimeoutActive: true, - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - equal := isMachineStatusEqual(data.action.s1, data.action.s2) - Expect(equal).To(Equal(data.expect.equal)) - }, - Entry("return true as status is same", &data{ - action: action{ - s1: machinev1.MachineStatus{ - LastOperation: lastOperation, - CurrentStatus: currentStatus, - }, - s2: machinev1.MachineStatus{ - LastOperation: lastOperation, - CurrentStatus: currentStatus, - }, - }, - expect: expect{ - equal: true, - }, - }), - Entry("return false as status is not equal", &data{ - action: action{ - s1: machinev1.MachineStatus{ - LastOperation: lastOperation, - CurrentStatus: currentStatus, - }, - s2: machinev1.MachineStatus{ - LastOperation: machinev1.LastOperation{ - Description: "test operation dummy", - LastUpdateTime: metav1.Now(), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationDelete, - }, - CurrentStatus: machinev1.CurrentStatus{ - LastUpdateTime: lastOperation.LastUpdateTime, - Phase: machinev1.MachinePending, - TimeoutActive: true, - }, - }, - }, - expect: expect{ - equal: false, - }, - }), - Entry("return true as only description is not same", &data{ - action: action{ - s1: machinev1.MachineStatus{ - LastOperation: lastOperation, - CurrentStatus: currentStatus, - }, - s2: machinev1.MachineStatus{ - LastOperation: machinev1.LastOperation{ - Description: "test operation dummy dummy", - LastUpdateTime: metav1.Now(), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - }, - CurrentStatus: machinev1.CurrentStatus{ - LastUpdateTime: lastOperation.LastUpdateTime, - Phase: machinev1.MachinePending, - TimeoutActive: true, - }, - }, - }, - expect: expect{ - equal: true, - }, - }), - ) - }) - - Describe("#validateMachineClass", func() { - type setup struct { - aws []*machinev1.AWSMachineClass - secrets []*corev1.Secret - } - type expect struct { - machineClass interface{} - secret *corev1.Secret - err bool - } - type data struct { - setup setup - action *machinev1.ClassSpec - expect expect - } - - objMeta := &metav1.ObjectMeta{ - GenerateName: "class", - Namespace: testNamespace, - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.aws { - machineObjects = append(machineObjects, o) - } - - coreObjects := []runtime.Object{} - for _, o := range data.setup.secrets { - coreObjects = append(coreObjects, o) - } - - controller, trackers := createController(stop, objMeta.Namespace, machineObjects, nil, coreObjects) - defer trackers.Stop() - - waitForCacheSync(stop, controller) - machineClass, secret, err := controller.validateMachineClass(data.action) - - if data.expect.machineClass == nil { - Expect(machineClass).To(BeNil()) - } else { - Expect(machineClass).To(Equal(data.expect.machineClass)) - } - if data.expect.secret == nil { - Expect(secret).To(BeNil()) - } else { - Expect(secret).To(Equal(data.expect.secret)) - } - if !data.expect.err { - Expect(err).To(BeNil()) - } else { - Expect(err).To(HaveOccurred()) - } - }, - Entry("non-existing machine class", &data{ - setup: setup{ - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - }, - action: &machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "non-existing", - }, - expect: expect{ - err: true, - }, - }), - Entry("non-existing secret", &data{ - setup: setup{ - secrets: []*corev1.Secret{}, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - }, - action: &machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "class-0", - }, - expect: expect{ - machineClass: &machinev1.AWSMachineClass{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - err: false, //TODO Why? Create issue - }, - }), - Entry("valid", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - }, - action: &machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "class-0", - }, - expect: expect{ - machineClass: &machinev1.AWSMachineClass{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - err: false, - }, - }), - ) - }) - - Describe("#machineCreate", func() { - type setup struct { - secrets []*corev1.Secret - aws []*machinev1.AWSMachineClass - openstack []*machinev1.OpenStackMachineClass - machines []*machinev1.Machine - fakeResourceActions *customfake.ResourceActions - } - type action struct { - machine string - fakeProviderID string - fakeNodeName string - fakeError error - } - type expect struct { - machine *machinev1.Machine - err bool - } - type data struct { - setup setup - action action - expect expect - } - objMeta := &metav1.ObjectMeta{ - GenerateName: "machine", - Namespace: "test", - } - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.aws { - machineObjects = append(machineObjects, o) - } - for _, o := range data.setup.openstack { - machineObjects = append(machineObjects, o) - } - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - coreObjects := []runtime.Object{} - for _, o := range data.setup.secrets { - coreObjects = append(coreObjects, o) - } - - controller, trackers := createController(stop, objMeta.Namespace, machineObjects, nil, coreObjects) - defer trackers.Stop() - - waitForCacheSync(stop, controller) - - action := data.action - machine, err := controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - if data.setup.fakeResourceActions != nil { - trackers.ControlMachine.SetFakeResourceActions(data.setup.fakeResourceActions, 1) - } - - err = controller.machineCreate(context.TODO(), machine, driver.NewFakeDriver( - func() (string, string, error) { - return action.fakeProviderID, action.fakeNodeName, action.fakeError - }, - func(string, string) error { - return nil - }, - func(string) error { - return action.fakeError - }, nil, - func() (driver.VMs, error) { - return map[string]string{}, nil - }, nil, nil, nil, - )) - - if data.expect.err { - Expect(err).To(HaveOccurred()) - return - } - - Expect(err).To(BeNil()) - actual, err := controller.controlMachineClient.Machines(machine.Namespace).Get(context.TODO(), machine.Name, metav1.GetOptions{}) - Expect(err).To(BeNil()) - Expect(actual.Spec).To(Equal(data.expect.machine.Spec)) - Expect(actual.Status.Node).To(Equal(data.expect.machine.Status.Node)) - //TODO Conditions - }, - Entry("OpenStackSimple", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - openstack: []*machinev1.OpenStackMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.OpenStackMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "OpenStackMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: fmt.Errorf("Test Error"), - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "OpenStackMachineClass", - Name: "machine-0", - }, - }, - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: "Failed", - }, - LastOperation: machinev1.LastOperation{ - Description: "Cloud provider message - Test Error", - }, - }, nil, nil, nil), - err: true, - }, - }), - Entry("AWSSimple", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - ProviderID: "fakeID", - }, - }, &machinev1.MachineStatus{ - Node: "fakeNode", - //TODO conditions - }, nil, nil, nil), - err: false, - }, - }), - Entry("Machine creation success even on temporary APIServer disruption", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Machine: customfake.Actions{ - Get: apierrors.NewGenericServerResponse( - http.StatusBadRequest, - "dummy method", - schema.GroupResource{}, - "dummy name", - "Failed to GET machine", - 30, - true, - ), - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - ProviderID: "fakeID", - }, - }, &machinev1.MachineStatus{ - Node: "fakeNode", - //TODO conditions - }, nil, nil, nil), - err: false, - }, - }), - Entry("Orphan VM deletion on failing to find referred machine object", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Machine: customfake.Actions{ - Get: apierrors.NewGenericServerResponse( - http.StatusNotFound, - "dummy method", - schema.GroupResource{}, - "dummy name", - "Machine not found", - 30, - true, - ), - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - }, - expect: expect{ - err: true, - }, - }), - Entry("If ProviderID is available and node-name missing, ProviderID should be set back on machine object", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Machine: customfake.Actions{ - Get: apierrors.NewGenericServerResponse( - http.StatusBadRequest, - "dummy method", - schema.GroupResource{}, - "dummy name", - "Failed to GET machine", - 30, - true, - ), - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "providerid-0", - fakeNodeName: "", - fakeError: nil, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - ProviderID: "providerid", - }, - }, &machinev1.MachineStatus{ - Node: "", - //TODO conditions - }, nil, nil, nil), - err: false, - }, - }), - ) - }) - - Describe("#machineDelete", func() { - type setup struct { - secrets []*corev1.Secret - aws []*machinev1.AWSMachineClass - machines []*machinev1.Machine - fakeResourceActions *customfake.ResourceActions - } - type action struct { - machine string - fakeProviderID string - fakeNodeName string - nodeRecentlyNotReady *bool - fakeDriverGetVMs func() (driver.VMs, error) - fakeError error - forceDeleteLabelPresent bool - fakeMachineStatus *machinev1.MachineStatus - } - type expect struct { - machine *machinev1.Machine - errOccurred bool - machineDeleted bool - nodeDeleted bool - } - type data struct { - setup setup - action action - expect expect - } - objMeta := &metav1.ObjectMeta{ - GenerateName: "machine", - Namespace: "test", - } - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.aws { - machineObjects = append(machineObjects, o) - } - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - coreObjects := []runtime.Object{} - for _, o := range data.setup.secrets { - coreObjects = append(coreObjects, o) - } - - controller, trackers := createController(stop, objMeta.Namespace, machineObjects, nil, coreObjects) - defer trackers.Stop() - waitForCacheSync(stop, controller) - - action := data.action - machine, err := controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - fakeDriverGetVMsTemp := func() (driver.VMs, error) { return nil, nil } - - if action.fakeDriverGetVMs != nil { - fakeDriverGetVMsTemp = action.fakeDriverGetVMs - } - - fakeDriver := driver.NewFakeDriver( - func() (string, string, error) { - _, err := controller.targetCoreClient.CoreV1().Nodes().Create(context.TODO(), &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: action.fakeNodeName, - }, - }, metav1.CreateOptions{}) - if err != nil { - return "", "", err - } - return action.fakeProviderID, action.fakeNodeName, action.fakeError - }, - func(string, string) error { - return nil - }, - func(string) error { - return nil - }, - func() (string, error) { - return action.fakeProviderID, action.fakeError - }, - fakeDriverGetVMsTemp, - nil, nil, nil, - ) - - // Create a machine that is to be deleted later - err = controller.machineCreate(context.TODO(), machine, fakeDriver) - Expect(err).ToNot(HaveOccurred()) - - // Add finalizers - controller.addMachineFinalizers(context.TODO(), machine) - - // Fetch the latest machine version - machine, err = controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - if data.action.forceDeleteLabelPresent { - // Add labels for force deletion - clone := machine.DeepCopy() - clone.Labels["force-deletion"] = "True" - machine, err = controller.controlMachineClient.Machines(objMeta.Namespace).Update(context.TODO(), clone, metav1.UpdateOptions{}) - Expect(err).ToNot(HaveOccurred()) - } - - if data.action.fakeMachineStatus != nil { - clone := machine.DeepCopy() - clone.Status = *data.action.fakeMachineStatus - machine, err = controller.controlMachineClient.Machines(objMeta.Namespace).UpdateStatus(context.TODO(), clone, metav1.UpdateOptions{}) - Expect(err).ToNot(HaveOccurred()) - } - - if data.action.nodeRecentlyNotReady != nil { - node, nodeErr := controller.targetCoreClient.CoreV1().Nodes().Get(context.TODO(), machine.Status.Node, metav1.GetOptions{}) - Expect(nodeErr).To(Not(HaveOccurred())) - clone := node.DeepCopy() - newNodeCondition := corev1.NodeCondition{ - Type: v1.NodeReady, - Status: corev1.ConditionUnknown, - } - - if *data.action.nodeRecentlyNotReady { - newNodeCondition.LastTransitionTime = metav1.Time{Time: time.Now()} - } else { - newNodeCondition.LastTransitionTime = metav1.Time{Time: time.Now().Add(-time.Hour)} - } - - clone.Status.Conditions = []corev1.NodeCondition{newNodeCondition} - _, updateErr := controller.targetCoreClient.CoreV1().Nodes().UpdateStatus(context.TODO(), clone, metav1.UpdateOptions{}) - Expect(updateErr).To(BeNil()) - } - - if data.setup.fakeResourceActions != nil { - trackers.TargetCore.SetFakeResourceActions(data.setup.fakeResourceActions, math.MaxInt32) - } - - // Deletion of machine is triggered - err = controller.machineDelete(context.TODO(), machine, fakeDriver) - if data.expect.errOccurred { - Expect(err).To(HaveOccurred()) - } else { - Expect(err).ToNot(HaveOccurred()) - } - - node, nodeErr := controller.targetCoreClient.CoreV1().Nodes().Get(context.TODO(), machine.Status.Node, metav1.GetOptions{}) - machine, machineErr := controller.controlMachineClient.Machines(machine.Namespace).Get(context.TODO(), machine.Name, metav1.GetOptions{}) - - if data.expect.machineDeleted { - Expect(machineErr).To(HaveOccurred()) - } else { - Expect(machineErr).ToNot(HaveOccurred()) - Expect(machine).ToNot(BeNil()) - } - - if data.expect.nodeDeleted { - Expect(nodeErr).To(HaveOccurred()) - } else { - Expect(nodeErr).ToNot(HaveOccurred()) - Expect(node).ToNot(BeNil()) - } - }, - Entry("Simple machine deletion", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Allow proper deletion of the machine object when providerID is missing but actual VM still exists in cloud", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - //fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - fakeDriverGetVMs: func() (driver.VMs, error) { - return map[string]string{"fakeID-0": "machine-0"}, nil - }, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine deletion when drain fails", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Node: customfake.Actions{ - Update: "Failed to update nodes", - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - }, - expect: expect{ - errOccurred: true, - machineDeleted: false, - nodeDeleted: false, - }, - }), - Entry("Machine force deletion label is present", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - forceDeleteLabelPresent: true, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine force deletion label is present and when drain call fails (APIServer call fails)", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Node: customfake.Actions{ - Update: "Failed to update nodes", - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - forceDeleteLabelPresent: true, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine deletion when timeout occurred", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - fakeMachineStatus: &machinev1.MachineStatus{ - Node: "fakeNode-0", - LastOperation: machinev1.LastOperation{ - Description: "Deleting machine from cloud provider", - State: v1alpha1.MachineStateProcessing, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - }, - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineTerminating, - TimeoutActive: false, - // Updating last update time to 30 minutes before now - LastUpdateTime: metav1.NewTime(time.Now().Add(-30 * time.Minute)), - }, - }, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine deletion when last drain failed", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - fakeMachineStatus: &machinev1.MachineStatus{ - Node: "fakeNode-0", - LastOperation: machinev1.LastOperation{ - Description: "Drain failed - for random reason", - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - }, - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineTerminating, - TimeoutActive: false, - LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Minute)), - }, - }, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine deletion when last drain failed and current drain call also fails (APIServer call fails)", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Node: customfake.Actions{ - Update: "Failed to update nodes", - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - fakeMachineStatus: &machinev1.MachineStatus{ - Node: "fakeNode-0", - LastOperation: machinev1.LastOperation{ - Description: "Drain failed - for random reason", - State: v1alpha1.MachineStateFailed, - Type: v1alpha1.MachineOperationDelete, - LastUpdateTime: metav1.Now(), - }, - CurrentStatus: machinev1.CurrentStatus{ - Phase: v1alpha1.MachineTerminating, - TimeoutActive: false, - LastUpdateTime: metav1.NewTime(time.Now().Add(-2 * time.Minute)), - }, - }, - }, - expect: expect{ - errOccurred: true, - machineDeleted: false, - nodeDeleted: false, - }, - }), - Entry("Machine force deletion if underlying Node is NotReady for a long time", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - nodeRecentlyNotReady: func() *bool { ret := false; return &ret }(), - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: true, - }, - }), - Entry("Machine do not force deletion if underlying Node is NotReady for a small period of time, a Machine deletion fails, since kubelet fails to evict Pods", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - fakeResourceActions: &customfake.ResourceActions{ - Node: customfake.Actions{ - Update: "Failed to update nodes", - }, - }, - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "fakeNode-0", - fakeError: nil, - nodeRecentlyNotReady: func() *bool { ret := true; return &ret }(), - }, - expect: expect{ - errOccurred: true, - machineDeleted: false, - nodeDeleted: false, - }, - }), - Entry("Allow machine object deletion where nodeName doesn't exist", &data{ - setup: setup{ - secrets: []*corev1.Secret{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - }, - aws: []*machinev1.AWSMachineClass{ - { - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.AWSMachineClassSpec{ - SecretRef: newSecretReference(objMeta, 0), - }, - }, - }, - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "machine-0", - }, - }, - }, nil, nil, nil, nil), - }, - action: action{ - machine: "machine-0", - fakeProviderID: "fakeID-0", - fakeNodeName: "", //NodeName is set to emptyString - fakeError: nil, - }, - expect: expect{ - errOccurred: false, - machineDeleted: true, - nodeDeleted: false, - }, - }), - ) - }) - - Describe("#checkMachineTimeout", func() { - type setup struct { - machines []*machinev1.Machine - } - type action struct { - machine string - } - type expect struct { - machine *machinev1.Machine - err bool - } - type data struct { - setup setup - action action - expect expect - } - objMeta := &metav1.ObjectMeta{ - GenerateName: "machine", - Namespace: "test", - } - - machineName := "machine-0" - timeOutOccurred := -21 * time.Minute - timeOutNotOccurred := -5 * time.Minute - creationTimeOut := 20 * time.Minute - healthTimeOut := 10 * time.Minute - - DescribeTable("##Machine Timeout Scenarios", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - coreObjects := []runtime.Object{} - - controller, trackers := createController(stop, objMeta.Namespace, machineObjects, nil, coreObjects) - defer trackers.Stop() - waitForCacheSync(stop, controller) - - action := data.action - machine, err := controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - //Expect(err).ToNot(HaveOccurred()) - - controller.checkMachineTimeout(context.TODO(), machine) - - actual, err := controller.controlMachineClient.Machines(machine.Namespace).Get(context.TODO(), machine.Name, metav1.GetOptions{}) - Expect(err).To(BeNil()) - Expect(actual.Status.CurrentStatus.Phase).To(Equal(data.expect.machine.Status.CurrentStatus.Phase)) - Expect(actual.Status.CurrentStatus.TimeoutActive).To(Equal(data.expect.machine.Status.CurrentStatus.TimeoutActive)) - Expect(actual.Status.LastOperation.Description).To(Equal(data.expect.machine.Status.LastOperation.Description)) - Expect(actual.Status.LastOperation.State).To(Equal(data.expect.machine.Status.LastOperation.State)) - Expect(actual.Status.LastOperation.Type).To(Equal(data.expect.machine.Status.LastOperation.Type)) - }, - Entry("Machine is still running", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineRunning, - TimeoutActive: false, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutNotOccurred)), - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine % successfully joined the cluster", machineName), - State: machinev1.MachineStateSuccessful, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutNotOccurred)), - }, - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineRunning, - TimeoutActive: false, - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine % successfully joined the cluster", machineName), - State: machinev1.MachineStateSuccessful, - Type: machinev1.MachineOperationCreate, - }, - }, nil, nil, nil), - }, - }), - Entry("Machine creation has still not timed out", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineUnknown, - TimeoutActive: true, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutNotOccurred)), - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine %s is unhealthy - changing MachineState to Unknown", machineName), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutNotOccurred)), - }, - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineUnknown, - TimeoutActive: true, - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine %s is unhealthy - changing MachineState to Unknown", machineName), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - }, - }, nil, nil, nil), - }, - }), - Entry("Machine creation has timed out", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachinePending, - TimeoutActive: true, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutOccurred)), - }, - LastOperation: machinev1.LastOperation{ - Description: "Creating machine on cloud provider", - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutOccurred)), - }, - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineFailed, - TimeoutActive: false, - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf( - "Machine %s failed to join the cluster in %s minutes.", - machineName, - creationTimeOut, - ), - State: machinev1.MachineStateFailed, - Type: machinev1.MachineOperationCreate, - }, - }, nil, nil, nil), - }, - }), - Entry("Machine health has timed out", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineUnknown, - TimeoutActive: true, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutOccurred)), - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine %s is unhealthy - changing MachineState to Unknown", machineName), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationHealthCheck, - LastUpdateTime: metav1.NewTime(time.Now().Add(timeOutOccurred)), - }, - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineFailed, - TimeoutActive: false, - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf( - "Machine %s is not healthy since %s minutes. Changing status to failed. Node Conditions: %+v", - machineName, - healthTimeOut, - []corev1.NodeCondition{}, - ), - State: machinev1.MachineStateFailed, - Type: machinev1.MachineOperationHealthCheck, - }, - }, nil, nil, nil), - }, - }), - ) - }) - - Describe("#updateMachineState", func() { - type setup struct { - machines []*machinev1.Machine - nodes []*corev1.Node - } - type action struct { - machine string - } - type expect struct { - machine *machinev1.Machine - err bool - } - type data struct { - setup setup - action action - expect expect - } - objMeta := &metav1.ObjectMeta{ - GenerateName: "machine", - // using default namespace for non-namespaced objects - // as our current fake client is with the assumption - // that all objects are namespaced - Namespace: "test", - } - - machineName := "machine-0" - - DescribeTable("##Different machine state update scenarios", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - machineObjects := []runtime.Object{} - for _, o := range data.setup.machines { - machineObjects = append(machineObjects, o) - } - - coreObjects := []runtime.Object{} - for _, o := range data.setup.nodes { - coreObjects = append(coreObjects, o) - } - - controller, trackers := createController(stop, objMeta.Namespace, machineObjects, nil, coreObjects) - defer trackers.Stop() - waitForCacheSync(stop, controller) - - action := data.action - machine, err := controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - controller.updateMachineState(context.TODO(), machine) - node, _ := controller.targetCoreClient.CoreV1().Nodes().Get(context.TODO(), "machine-0", metav1.GetOptions{}) - klog.Error(node) - - actual, err := controller.controlMachineClient.Machines(objMeta.Namespace).Get(context.TODO(), action.machine, metav1.GetOptions{}) - - Expect(err).To(BeNil()) - Expect(actual.Name).To(Equal(data.expect.machine.Name)) - Expect(actual.Status.Node).To(Equal(data.expect.machine.Status.Node)) - Expect(actual.Status.CurrentStatus.Phase).To(Equal(data.expect.machine.Status.CurrentStatus.Phase)) - Expect(actual.Status.CurrentStatus.TimeoutActive).To(Equal(data.expect.machine.Status.CurrentStatus.TimeoutActive)) - Expect(actual.Status.LastOperation.State).To(Equal(data.expect.machine.Status.LastOperation.State)) - Expect(actual.Status.LastOperation.Type).To(Equal(data.expect.machine.Status.LastOperation.Type)) - Expect(actual.Status.LastOperation.Description).To(Equal(data.expect.machine.Status.LastOperation.Description)) - - if data.expect.machine.Labels != nil { - if _, ok := data.expect.machine.Labels["node"]; ok { - Expect(actual.Labels["node"]).To(Equal(data.expect.machine.Labels["node"])) - } - } - - for i := range actual.Status.Conditions { - Expect(actual.Status.Conditions[i].Type).To(Equal(data.expect.machine.Status.Conditions[i].Type)) - Expect(actual.Status.Conditions[i].Status).To(Equal(data.expect.machine.Status.Conditions[i].Status)) - Expect(actual.Status.Conditions[i].Reason).To(Equal(data.expect.machine.Status.Conditions[i].Reason)) - Expect(actual.Status.Conditions[i].Message).To(Equal(data.expect.machine.Status.Conditions[i].Message)) - } - }, - Entry("Machine does not have a node backing", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{}, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{}, nil, nil, nil), - }, - }), - Entry("Node object backing machine not found and machine conditions are empty", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "dummy-node", - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "dummy-node", - }, nil, nil, nil), - }, - }), - Entry("Machine is running but node object is lost", &data{ - setup: setup{ - machines: newMachines( - 1, - &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, - &machinev1.MachineStatus{ - Node: "dummy-node", - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineRunning, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf("Machine % successfully joined the cluster", machineName), - State: machinev1.MachineStateSuccessful, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - }, - Conditions: []corev1.NodeCondition{ - { - Message: "kubelet is posting ready status", - Reason: "KubeletReady", - Status: "True", - Type: "Ready", - }, - }, - }, nil, nil, nil), - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "dummy-node", - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineUnknown, - TimeoutActive: true, - LastUpdateTime: metav1.Now(), - }, - LastOperation: machinev1.LastOperation{ - Description: fmt.Sprintf( - "Node object went missing. Machine %s is unhealthy - changing MachineState to Unknown", - machineName, - ), - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationHealthCheck, - LastUpdateTime: metav1.Now(), - }, - Conditions: []corev1.NodeCondition{ - { - Message: "kubelet is posting ready status", - Reason: "KubeletReady", - Status: "True", - Type: "Ready", - }, - }, - }, nil, nil, nil), - }, - }), - Entry("Machine and node both are present and kubelet ready status is updated", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "machine", - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachinePending, - TimeoutActive: true, - LastUpdateTime: metav1.Now(), - }, - LastOperation: machinev1.LastOperation{ - Description: "Creating machine on cloud provider", - State: machinev1.MachineStateProcessing, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - }, - Conditions: []corev1.NodeCondition{ - { - Message: "kubelet is not ready", - Reason: "KubeletReady", - Status: "False", - Type: "Ready", - }, - }, - }, nil, nil, nil), - nodes: []*corev1.Node{ - { - TypeMeta: metav1.TypeMeta{ - Kind: "Node", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-0", - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - { - Message: "kubelet is posting ready status", - Reason: "KubeletReady", - Status: "True", - Type: "Ready", - }, - }, - }, - }, - }, - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "machine", - CurrentStatus: machinev1.CurrentStatus{ - Phase: machinev1.MachineRunning, - TimeoutActive: false, - LastUpdateTime: metav1.Now(), - }, - LastOperation: machinev1.LastOperation{ - Description: "Machine machine-0 successfully joined the cluster", - State: machinev1.MachineStateSuccessful, - Type: machinev1.MachineOperationCreate, - LastUpdateTime: metav1.Now(), - }, - Conditions: []corev1.NodeCondition{ - { - Message: "kubelet is posting ready status", - Reason: "KubeletReady", - Status: "True", - Type: "Ready", - }, - }, - }, nil, nil, nil), - }, - }), - Entry("Machine object does not have node-label and node exists", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - }, &machinev1.MachineStatus{ - Node: "node", - }, nil, nil, nil), - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-0", - }, - }, - }, - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-0", - }, - }, &machinev1.MachineStatus{ - Node: "node", - }, nil, nil, - map[string]string{ - "node": "node-0", - }, - ), - }, - }), - Entry("Machine object does not have status.node set and node exists then it should adopt node using providerID", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - ProviderID: "aws//fakeID", - }, - }, &machinev1.MachineStatus{}, nil, nil, nil), - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-0", - }, - Spec: corev1.NodeSpec{ - ProviderID: "aws//fakeID-0", - }, - }, - }, - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-0", - }, - }, &machinev1.MachineStatus{ - Node: "node", - }, nil, nil, - map[string]string{ - "node": "node-0", - }, - ), - }, - }), - Entry("Machine object does not have status.node set and node exists (without providerID) then it should not adopt node using providerID", &data{ - setup: setup{ - machines: newMachines(1, &machinev1.MachineTemplateSpec{ - ObjectMeta: *newObjectMeta(objMeta, 0), - Spec: machinev1.MachineSpec{ - ProviderID: "aws//fakeID", - }, - }, &machinev1.MachineStatus{}, nil, nil, nil), - nodes: []*corev1.Node{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "node-0", - }, - Spec: corev1.NodeSpec{ - ProviderID: "", - }, - }, - }, - }, - action: action{ - machine: machineName, - }, - expect: expect{ - machine: newMachine(&machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "machine-0", - }, - }, nil, nil, nil, nil, - ), - }, - }), - ) - }) - -}) diff --git a/pkg/controller/machine_util.go b/pkg/controller/machine_util.go index 0907eca0e..5d73dfc4e 100644 --- a/pkg/controller/machine_util.go +++ b/pkg/controller/machine_util.go @@ -24,39 +24,17 @@ package controller import ( "context" - "encoding/json" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/nodeops" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" - machineapi "github.com/gardener/machine-controller-manager/pkg/apis/machine" "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" v1alpha1client "github.com/gardener/machine-controller-manager/pkg/client/clientset/versioned/typed/machine/v1alpha1" v1alpha1listers "github.com/gardener/machine-controller-manager/pkg/client/listers/machine/v1alpha1" - v1 "k8s.io/api/core/v1" errorsutil "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/util/retry" ) -const ( - // LastAppliedALTAnnotation contains the last configuration of annotations, labels & taints applied on the node object - LastAppliedALTAnnotation = "node.machine.sapcloud.io/last-applied-anno-labels-taints" - // NodeTerminationCondition describes nodes that are terminating - NodeTerminationCondition = "Termination" - // NodeUnhealthy is a node termination reason for failed machines - NodeUnhealthy = "Unhealthy" - // NodeScaledDown is a node termination reason for healthy deleted machines - NodeScaledDown = "ScaleDown" -) - -var ( - // emptyMap is a dummy emptyMap to compare with - emptyMap = make(map[string]string) -) - // TODO: use client library instead when it starts to support update retries // see https://github.com/kubernetes/kubernetes/issues/21479 type updateMachineFunc func(machine *v1alpha1.Machine) error @@ -90,502 +68,3 @@ func UpdateMachineWithRetries(ctx context.Context, machineClient v1alpha1client. return machine, retryErr } - -func (c *controller) validateMachineClass(classSpec *v1alpha1.ClassSpec) (interface{}, map[string][]byte, error) { - - var MachineClass interface{} - var secretData map[string][]byte - - switch classSpec.Kind { - case AWSMachineClassKind: - AWSMachineClass, err := c.awsMachineClassLister.AWSMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("AWSMachineClass %q/%q not found. Skipping. %v", c.namespace, classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = AWSMachineClass - - // Validate AWSMachineClass - internalAWSMachineClass := &machineapi.AWSMachineClass{} - err = c.internalExternalScheme.Convert(AWSMachineClass, internalAWSMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidateAWSMachineClass(internalAWSMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of AWSMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(AWSMachineClass.Name, AWSMachineClass.Spec.SecretRef, AWSMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - - case AzureMachineClassKind: - AzureMachineClass, err := c.azureMachineClassLister.AzureMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("AzureMachineClass %q not found. Skipping. %v", classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = AzureMachineClass - - // Validate AzureMachineClass - internalAzureMachineClass := &machineapi.AzureMachineClass{} - err = c.internalExternalScheme.Convert(AzureMachineClass, internalAzureMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidateAzureMachineClass(internalAzureMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of AzureMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(AzureMachineClass.Name, AzureMachineClass.Spec.SecretRef, AzureMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - - case GCPMachineClassKind: - GCPMachineClass, err := c.gcpMachineClassLister.GCPMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("GCPMachineClass %q not found. Skipping. %v", classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = GCPMachineClass - - // Validate GCPMachineClass - internalGCPMachineClass := &machineapi.GCPMachineClass{} - err = c.internalExternalScheme.Convert(GCPMachineClass, internalGCPMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidateGCPMachineClass(internalGCPMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of GCPMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(GCPMachineClass.Name, GCPMachineClass.Spec.SecretRef, GCPMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - case OpenStackMachineClassKind: - OpenStackMachineClass, err := c.openStackMachineClassLister.OpenStackMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("OpenStackMachineClass %q not found. Skipping. %v", classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = OpenStackMachineClass - - // Validate OpenStackMachineClass - internalOpenStackMachineClass := &machineapi.OpenStackMachineClass{} - err = c.internalExternalScheme.Convert(OpenStackMachineClass, internalOpenStackMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidateOpenStackMachineClass(internalOpenStackMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of OpenStackMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(OpenStackMachineClass.Name, OpenStackMachineClass.Spec.SecretRef, OpenStackMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - - case AlicloudMachineClassKind: - AlicloudMachineClass, err := c.alicloudMachineClassLister.AlicloudMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("AlicloudMachineClass %q/%q not found. Skipping. %v", c.namespace, classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = AlicloudMachineClass - - // Validate AlicloudMachineClass - internalAlicloudMachineClass := &machineapi.AlicloudMachineClass{} - err = c.internalExternalScheme.Convert(AlicloudMachineClass, internalAlicloudMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidateAlicloudMachineClass(internalAlicloudMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of AlicloudMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(AlicloudMachineClass.Name, AlicloudMachineClass.Spec.SecretRef, AlicloudMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - - case PacketMachineClassKind: - PacketMachineClass, err := c.packetMachineClassLister.PacketMachineClasses(c.namespace).Get(classSpec.Name) - if err != nil { - klog.V(2).Infof("PacketMachineClass %q/%q not found. Skipping. %v", c.namespace, classSpec.Name, err) - return MachineClass, nil, err - } - MachineClass = PacketMachineClass - - // Validate AlicloudMachineClass - internalPacketMachineClass := &machineapi.PacketMachineClass{} - err = c.internalExternalScheme.Convert(PacketMachineClass, internalPacketMachineClass, nil) - if err != nil { - klog.V(2).Info("Error in scheme conversion") - return MachineClass, nil, err - } - - validationerr := validation.ValidatePacketMachineClass(internalPacketMachineClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.V(2).Infof("Validation of PacketMachineClass failed %s", validationerr.ToAggregate().Error()) - return MachineClass, nil, nil - } - - // Get secret data - secretData, err = c.getSecretData(PacketMachineClass.Name, PacketMachineClass.Spec.SecretRef, PacketMachineClass.Spec.CredentialsSecretRef) - if err != nil { - klog.V(2).Infof("Could not compute secret data: %+v", err) - return MachineClass, nil, err - } - - default: - klog.V(2).Infof("ClassKind %q not found. Machine maybe be processed by external controller", classSpec.Kind) - } - - return MachineClass, secretData, nil -} - -// nodeConditionsHaveChanged compares two node statuses to see if any of the statuses have changed -func nodeConditionsHaveChanged(machineConditions []v1.NodeCondition, nodeConditions []v1.NodeCondition) bool { - - if len(machineConditions) != len(nodeConditions) { - return true - } - - for i := range nodeConditions { - if nodeConditions[i].Status != machineConditions[i].Status { - return true - } - } - - return false -} - -// syncMachineNodeTemplate syncs nodeTemplates between machine and corresponding node-object. -// It ensures, that any nodeTemplate element available on Machine should be available on node-object. -// Although there could be more elements already available on node-object which will not be touched. -func (c *controller) syncMachineNodeTemplates(ctx context.Context, machine *v1alpha1.Machine) error { - var ( - initializedNodeAnnotation = false - lastAppliedALT v1alpha1.NodeTemplateSpec - currentlyAppliedALTJSONByte []byte - ) - - node, err := c.nodeLister.Get(machine.Status.Node) - if err != nil || node == nil { - klog.Errorf("Error: Could not get the node-object or node-object is missing - err: %q", err) - // Dont return error so that other steps can be executed. - return nil - } - nodeCopy := node.DeepCopy() - - // Initialize node annotations if empty - if nodeCopy.Annotations == nil { - nodeCopy.Annotations = make(map[string]string) - initializedNodeAnnotation = true - } - - // Extracts the last applied annotations to lastAppliedLabels - lastAppliedALTJSONString, exists := node.Annotations[LastAppliedALTAnnotation] - if exists { - err = json.Unmarshal([]byte(lastAppliedALTJSONString), &lastAppliedALT) - if err != nil { - klog.Errorf("Error occurred while syncing node annotations, labels & taints: %s", err) - return err - } - } - - annotationsChanged := SyncMachineAnnotations(machine, nodeCopy, lastAppliedALT.Annotations) - labelsChanged := SyncMachineLabels(machine, nodeCopy, lastAppliedALT.Labels) - taintsChanged := SyncMachineTaints(machine, nodeCopy, lastAppliedALT.Spec.Taints) - - // Update node-object with latest nodeTemplate elements if elements have changed. - if initializedNodeAnnotation || labelsChanged || annotationsChanged || taintsChanged { - - klog.V(2).Infof( - "Updating machine annotations:%v, labels:%v, taints:%v for machine: %q", - annotationsChanged, - labelsChanged, - taintsChanged, - machine.Name, - ) - - // Update the LastAppliedALTAnnotation - lastAppliedALT = machine.Spec.NodeTemplateSpec - currentlyAppliedALTJSONByte, err = json.Marshal(lastAppliedALT) - if err != nil { - klog.Errorf("Error occurred while syncing node annotations, labels & taints: %s", err) - return err - } - nodeCopy.Annotations[LastAppliedALTAnnotation] = string(currentlyAppliedALTJSONByte) - - _, err := c.targetCoreClient.CoreV1().Nodes().Update(ctx, nodeCopy, metav1.UpdateOptions{}) - if err != nil { - return err - } - } - return nil -} - -// SyncMachineAnnotations syncs the annotations of the machine with node-objects. -// It returns true if update is needed else false. -func SyncMachineAnnotations( - machine *v1alpha1.Machine, - node *v1.Node, - lastAppliedAnnotations map[string]string, -) bool { - toBeUpdated := false - mAnnotations, nAnnotations := machine.Spec.NodeTemplateSpec.Annotations, node.Annotations - - // Initialize node annotations if nil - if nAnnotations == nil { - nAnnotations = make(map[string]string) - node.Annotations = nAnnotations - } - // Intialize machine annotations to empty map if nil - if mAnnotations == nil { - mAnnotations = emptyMap - } - - // Delete any annotation that existed in the past but has been deleted now - for lastAppliedAnnotationKey := range lastAppliedAnnotations { - if _, exists := mAnnotations[lastAppliedAnnotationKey]; !exists { - delete(nAnnotations, lastAppliedAnnotationKey) - toBeUpdated = true - } - } - - // Add/Update any key that doesn't exist or whose value as changed - for mKey, mValue := range mAnnotations { - if nValue, exists := nAnnotations[mKey]; !exists || mValue != nValue { - nAnnotations[mKey] = mValue - toBeUpdated = true - } - } - - return toBeUpdated -} - -// SyncMachineLabels syncs the labels of the machine with node-objects. -// It returns true if update is needed else false. -func SyncMachineLabels( - machine *v1alpha1.Machine, - node *v1.Node, - lastAppliedLabels map[string]string, -) bool { - toBeUpdated := false - mLabels, nLabels := machine.Spec.NodeTemplateSpec.Labels, node.Labels - - // Initialize node labels if nil - if nLabels == nil { - nLabels = make(map[string]string) - node.Labels = nLabels - } - // Intialize machine labels to empty map if nil - if mLabels == nil { - mLabels = emptyMap - } - - // Delete any labels that existed in the past but has been deleted now - for lastAppliedLabelKey := range lastAppliedLabels { - if _, exists := mLabels[lastAppliedLabelKey]; !exists { - delete(nLabels, lastAppliedLabelKey) - toBeUpdated = true - } - } - - // Add/Update any key that doesn't exist or whose value as changed - for mKey, mValue := range mLabels { - if nValue, exists := nLabels[mKey]; !exists || mValue != nValue { - nLabels[mKey] = mValue - toBeUpdated = true - } - } - - return toBeUpdated -} - -type taintKeyEffect struct { - // Required. The taint key to be applied to a node. - Key string - // Valid effects are NoSchedule, PreferNoSchedule and NoExecute. - Effect v1.TaintEffect -} - -// SyncMachineTaints syncs the taints of the machine with node-objects. -// It returns true if update is needed else false. -func SyncMachineTaints( - machine *v1alpha1.Machine, - node *v1.Node, - lastAppliedTaints []v1.Taint, -) bool { - toBeUpdated := false - mTaints, nTaints := machine.Spec.NodeTemplateSpec.Spec.Taints, node.Spec.Taints - mTaintsMap := make(map[taintKeyEffect]*v1.Taint) - nTaintsMap := make(map[taintKeyEffect]*v1.Taint) - - // Convert the slice of taints to map of taint [key, effect] = Taint - // Helps with indexed searching - for i := range mTaints { - mTaint := &mTaints[i] - taintKE := taintKeyEffect{ - Key: mTaint.Key, - Effect: mTaint.Effect, - } - mTaintsMap[taintKE] = mTaint - } - for i := range nTaints { - nTaint := &nTaints[i] - taintKE := taintKeyEffect{ - Key: nTaint.Key, - Effect: nTaint.Effect, - } - nTaintsMap[taintKE] = nTaint - } - - // Delete taints that existed on the machine object in the last update but deleted now - for _, lastAppliedTaint := range lastAppliedTaints { - - lastAppliedKE := taintKeyEffect{ - Key: lastAppliedTaint.Key, - Effect: lastAppliedTaint.Effect, - } - - if _, exists := mTaintsMap[lastAppliedKE]; !exists { - delete(nTaintsMap, lastAppliedKE) - toBeUpdated = true - } - } - - // Add any taints that exists in the machine object but not on the node object - for mKE, mV := range mTaintsMap { - if nV, exists := nTaintsMap[mKE]; !exists || *nV != *mV { - nTaintsMap[mKE] = mV - toBeUpdated = true - } - } - - if toBeUpdated { - // Convert the map of taints to slice of taints - nTaints = make([]v1.Taint, len(nTaintsMap)) - i := 0 - for _, nV := range nTaintsMap { - nTaints[i] = *nV - i++ - } - node.Spec.Taints = nTaints - } - - return toBeUpdated -} - -func (c *controller) UpdateNodeTerminationCondition(ctx context.Context, machine *v1alpha1.Machine) error { - if machine.Status.CurrentStatus.Phase == "" { - return nil - } - - nodeName := machine.Status.Node - - terminationCondition := v1.NodeCondition{ - Type: NodeTerminationCondition, - Status: v1.ConditionTrue, - LastHeartbeatTime: metav1.Now(), - LastTransitionTime: metav1.Now(), - } - - // check if condition already exists - cond, err := nodeops.GetNodeCondition(ctx, c.targetCoreClient, nodeName, NodeTerminationCondition) - if err != nil { - if apierrors.IsNotFound(err) { - return nil - } - return err - } - - if cond != nil && machine.Status.CurrentStatus.Phase == v1alpha1.MachineTerminating { - // do not consider machine terminating phase if node already terminating - terminationCondition.Reason = cond.Reason - terminationCondition.Message = cond.Message - } else { - setTerminationReasonByPhase(machine.Status.CurrentStatus.Phase, &terminationCondition) - } - - err = nodeops.AddOrUpdateConditionsOnNode(ctx, c.targetCoreClient, nodeName, terminationCondition) - if apierrors.IsNotFound(err) { - return nil - } - return err -} - -func (c *controller) getSecretData(machineClassName string, secretRefs ...*v1.SecretReference) (map[string][]byte, error) { - var secretData map[string][]byte - - for _, secretRef := range secretRefs { - if secretRef == nil { - continue - } - - secret, err := c.getSecret(secretRef, machineClassName) - if err != nil { - klog.V(2).Infof("Secret reference %s/%s not found", secretRef.Namespace, secretRef.Name) - return nil, err - } - secretData = mergeDataMaps(secretData, secret.Data) - } - - return secretData, nil -} - -func setTerminationReasonByPhase(phase v1alpha1.MachinePhase, terminationCondition *v1.NodeCondition) { - if phase == v1alpha1.MachineFailed { // if failed, terminated due to health - terminationCondition.Reason = NodeUnhealthy - terminationCondition.Message = "Machine Controller is terminating failed machine" - } else { // in all other cases (except for already terminating): assume scale down - terminationCondition.Reason = NodeScaledDown - terminationCondition.Message = "Machine Controller is scaling down machine" - } -} - -func mergeDataMaps(in map[string][]byte, maps ...map[string][]byte) map[string][]byte { - out := make(map[string][]byte) - - for _, m := range append([]map[string][]byte{in}, maps...) { - for k, v := range m { - out[k] = v - } - } - - return out -} diff --git a/pkg/controller/machine_util_test.go b/pkg/controller/machine_util_test.go deleted file mode 100644 index 5b648770e..000000000 --- a/pkg/controller/machine_util_test.go +++ /dev/null @@ -1,2168 +0,0 @@ -/* -Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package controller - -import ( - "context" - "encoding/json" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - machinev1 "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog/v2" -) - -var _ = Describe("machine_util", func() { - - Describe("#syncMachineNodeTemplates", func() { - - type setup struct { - machine *machinev1.Machine - } - type action struct { - node *corev1.Node - } - type expect struct { - node *corev1.Node - err bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - controlObjects := []runtime.Object{} - coreObjects := []runtime.Object{} - - machineObject := data.setup.machine - - nodeObject := data.action.node - coreObjects = append(coreObjects, nodeObject) - controlObjects = append(controlObjects, machineObject) - - c, trackers := createController(stop, testNamespace, controlObjects, nil, coreObjects) - defer trackers.Stop() - waitForCacheSync(stop, c) - - err := c.syncMachineNodeTemplates(context.TODO(), machineObject) - - waitForCacheSync(stop, c) - - if !data.expect.err { - Expect(err).To(BeNil()) - } else { - Expect(err).To(HaveOccurred()) - } - - //updatedNodeObject, _ := c.nodeLister.Get(nodeObject.Name) - updatedNodeObject, _ := c.targetCoreClient.CoreV1().Nodes().Get(context.TODO(), nodeObject.Name, metav1.GetOptions{}) - - if data.expect.node != nil { - Expect(updatedNodeObject.Spec.Taints).Should(ConsistOf(data.expect.node.Spec.Taints)) - Expect(updatedNodeObject.Labels).Should(Equal(data.expect.node.Labels)) - - // ignore LastAppliedALTAnnotataion - delete(updatedNodeObject.Annotations, LastAppliedALTAnnotation) - Expect(updatedNodeObject.Annotations).Should(Equal(data.expect.node.Annotations)) - } - }, - - Entry("when nodeTemplate is not updated in node-object", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "test-machine-class", - }, - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{{ - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - &machinev1.MachineStatus{ - Node: "test-node", - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - Labels: map[string]string{ - "key1": "value1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Namespace: testNamespace, - Annotations: map[string]string{ - "anno1": "anno1", - }, - Labels: map[string]string{ - "key1": "value1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - err: false, - }, - }), - - Entry("when nodeTemplate is updated in node-object", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "test-machine-class", - }, - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"},\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - &machinev1.MachineStatus{ - Node: "test-node", - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - err: false, - }, - }), - - Entry("when node object does not exist", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "test-machine-class", - }, - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - &machinev1.MachineStatus{ - Node: "test-node", - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{}, - }, - expect: expect{ - node: &corev1.Node{}, - err: false, // we should not return error if node-object does not exist to ensure rest of the steps are then executed. - }, - }), - - Entry("Multiple taints with same key and value added to taint", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - Class: machinev1.ClassSpec{ - Kind: "AWSMachineClass", - Name: "test-machine-class", - }, - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoExecute", - }, - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - &machinev1.MachineStatus{ - Node: "test-node", - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"},\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - Labels: map[string]string{ - "key1": "value1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{}, - }, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Namespace: testNamespace, - Annotations: map[string]string{ - "anno1": "anno1", - }, - Labels: map[string]string{ - "key1": "value1", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: "NoExecute", - }, - { - Key: "key1", - Value: "value1", - Effect: "NoSchedule", - }, - }, - }, - }, - err: false, - }, - }), - ) - - }) - - Describe("#SyncMachineLabels", func() { - - type setup struct{} - type action struct { - node *corev1.Node - machine *machinev1.Machine - } - type expect struct { - node *corev1.Node - labelsChanged bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - c, trackers := createController(stop, testNamespace, nil, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - testNode := data.action.node - testMachine := data.action.machine - expectedNode := data.expect.node - - var lastAppliedALT v1alpha1.NodeTemplateSpec - lastAppliedALTJSONString, exists := testNode.Annotations[LastAppliedALTAnnotation] - if exists { - err := json.Unmarshal([]byte(lastAppliedALTJSONString), &lastAppliedALT) - if err != nil { - klog.Errorf("Error occurred while syncing node annotations, labels & taints: %s", err) - } - } - - labelsChanged := SyncMachineLabels(testMachine, testNode, lastAppliedALT.Labels) - - waitForCacheSync(stop, c) - - Expect(testNode.Labels).Should(Equal(expectedNode.Labels)) - Expect(labelsChanged).To(Equal(data.expect.labelsChanged)) - }, - - Entry("when labels have not been updated", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - labelsChanged: false, - }, - }), - - Entry("when labels values are updated ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "valueChanged", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "valueChanged", - }, - }, - }, - labelsChanged: true, - }, - }), - - Entry("when new label keys are added ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - }, - labelsChanged: true, - }, - }), - - Entry("when label is deleted from machine object", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\", \"key2\":\"value2\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - labelsChanged: true, - }, - }), - - Entry("when labels values are updated manually on node object", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value2", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - labelsChanged: true, - }, - }), - - Entry("when new labels are added on node-object ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - "keyNew": "valueNew", - }, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - "keyNew": "valueNew", - }, - }, - }, - labelsChanged: false, - }, - }), - - Entry("when existing labels are deleted from node-object ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{}, - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"labels\":{\"key1\":\"value1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Labels: map[string]string{ - "key1": "value1", - "key2": "value2", - }, - }, - }, - labelsChanged: true, - }, - }), - ) - - }) - - Describe("#SyncMachineAnnotations", func() { - - type setup struct{} - type action struct { - node *corev1.Node - machine *machinev1.Machine - } - type expect struct { - node *corev1.Node - annotationsChanged bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - c, trackers := createController(stop, testNamespace, nil, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - testNode := data.action.node - testMachine := data.action.machine - expectedNode := data.expect.node - - var lastAppliedALT v1alpha1.NodeTemplateSpec - lastAppliedALTJSONString, exists := testNode.Annotations[LastAppliedALTAnnotation] - if exists { - err := json.Unmarshal([]byte(lastAppliedALTJSONString), &lastAppliedALT) - if err != nil { - klog.Errorf("Error occurred while syncing node annotations, labels & taints: %s", err) - } - } - - annotationsChanged := SyncMachineAnnotations(testMachine, testNode, lastAppliedALT.Annotations) - - waitForCacheSync(stop, c) - - // ignore LastAppliedALTAnnotation for comparison - delete(testNode.Annotations, LastAppliedALTAnnotation) - Expect(testNode.Annotations).Should(Equal(expectedNode.Annotations)) - Expect(annotationsChanged).To(Equal(data.expect.annotationsChanged)) - }, - - Entry("when annotations have not been updated", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - annotationsChanged: false, - }, - }), - - Entry("when annotations values are updated ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "annoChanged", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "annoChanged", - }, - }, - }, - annotationsChanged: true, - }, - }), - - Entry("when new annotation keys are added ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - "anno2": "anno2", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - "anno2": "anno2", - }, - }, - }, - annotationsChanged: true, - }, - }), - - Entry("when annotations are deleted ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - "anno2": "anno2", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\", \"anno2\":\"anno2\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{}, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - annotationsChanged: true, - }, - }), - - Entry("when annotations values are updated manually on node object", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno2", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - annotationsChanged: true, - }, - }), - - Entry("when new annotations are added on node-objects.", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - "annoNew": "annoNew", - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - "annoNew": "annoNew", - }, - }, - }, - annotationsChanged: false, - }, - }), - - Entry("when existing annotations are deleted from node-objects. ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null,\"annotations\":{\"anno1\":\"anno1\"}}}", - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - "anno1": "anno1", - "anno2": "anno2", - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - "anno2": "anno2", - }, - }, - }, - annotationsChanged: true, - }, - }), - ) - - }) - - Describe("#SyncMachineTaints", func() { - - type setup struct{} - type action struct { - node *corev1.Node - machine *machinev1.Machine - } - type expect struct { - node *corev1.Node - taintsChanged bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - c, trackers := createController(stop, testNamespace, nil, nil, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - testNode := data.action.node - testMachine := data.action.machine - expectedNode := data.expect.node - - var lastAppliedALT v1alpha1.NodeTemplateSpec - lastAppliedALTJSONString, exists := testNode.Annotations[LastAppliedALTAnnotation] - if exists { - err := json.Unmarshal([]byte(lastAppliedALTJSONString), &lastAppliedALT) - if err != nil { - klog.Errorf("Error occurred while syncing node annotations, labels & taints: %s", err) - } - } - - taintsChanged := SyncMachineTaints(testMachine, testNode, lastAppliedALT.Spec.Taints) - - waitForCacheSync(stop, c) - - Expect(testNode.Spec.Taints).Should(ConsistOf(expectedNode.Spec.Taints)) - Expect(taintsChanged).To(Equal(data.expect.taintsChanged)) - }, - - Entry("when taints have not been updated", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: false, - }, - }), - - Entry("when taints values are updated ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"OldValue\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "OldValue", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "NewValue", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "NewValue", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: true, - }, - }), - - Entry("when new taints are added ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key2", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key2", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: true, - }, - }), - - Entry("when taints are deleted ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"},{\"key\":\"Key2\",\"value\":\"Value2\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key2", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: true, - }, - }), - - Entry("when node taint value is overwritten manually & new taint was added with same key & value but different effect", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key1", - Value: "Value1", - Effect: "NoExecute", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key1", - Value: "Value1", - Effect: "NoExecute", - }, - }, - }, - }, - taintsChanged: true, - }, - }), - - Entry("when new taints are added on node-object ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "KeyNew", - Value: "ValueNew", - Effect: "NoSchedule", - }, - }, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "KeyNew", - Value: "ValueNew", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: false, - }, - }), - - Entry("when existing taints are deleted from node-objects ", &data{ - setup: setup{}, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - LastAppliedALTAnnotation: "{\"metadata\":{\"creationTimestamp\":null},\"spec\":{\"taints\":[{\"key\":\"Key1\",\"value\":\"Value1\",\"effect\":\"NoSchedule\"}]}}", - }, - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{}, - }, - }, - machine: newMachine( - &machinev1.MachineTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "test-label": "test-label", - }, - }, - Spec: machinev1.MachineSpec{ - NodeTemplateSpec: machinev1.NodeTemplateSpec{ - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key2", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - }, - }, - nil, nil, nil, nil), - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{ - Taints: []corev1.Taint{ - { - Key: "Key1", - Value: "Value1", - Effect: "NoSchedule", - }, - { - Key: "Key2", - Value: "Value2", - Effect: "NoSchedule", - }, - }, - }, - }, - taintsChanged: true, - }, - }), - ) - - }) - - Describe("#UpdateNodeTerminationCondition", func() { - - type setup struct { - machine *machinev1.Machine - } - type action struct { - node *corev1.Node - } - type expect struct { - node *corev1.Node - err bool - } - type data struct { - setup setup - action action - expect expect - } - - DescribeTable("##table", - func(data *data) { - stop := make(chan struct{}) - defer close(stop) - - controlObjects := []runtime.Object{} - coreObjects := []runtime.Object{} - - machineObject := data.setup.machine - - nodeObject := data.action.node - coreObjects = append(coreObjects, nodeObject) - controlObjects = append(controlObjects, machineObject) - - c, trackers := createController(stop, testNamespace, controlObjects, nil, coreObjects) - defer trackers.Stop() - waitForCacheSync(stop, c) - - err := c.UpdateNodeTerminationCondition(context.TODO(), machineObject) - - waitForCacheSync(stop, c) - - if !data.expect.err { - Expect(err).To(BeNil()) - } else { - Expect(err).To(HaveOccurred()) - } - - updatedNodeObject, _ := c.targetCoreClient.CoreV1().Nodes().Get(context.TODO(), nodeObject.Name, metav1.GetOptions{}) - - if data.expect.node != nil { - conditions := data.expect.node.Status.Conditions - if len(conditions) > 0 { - Expect(updatedNodeObject.Status.Conditions[0].Type).Should(Equal(data.expect.node.Status.Conditions[0].Type)) - Expect(updatedNodeObject.Status.Conditions[0].Status).Should(Equal(data.expect.node.Status.Conditions[0].Status)) - Expect(updatedNodeObject.Status.Conditions[0].Reason).Should(Equal(data.expect.node.Status.Conditions[0].Reason)) - } - } - }, - - Entry("when machine phase is failed", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineFailed}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeUnhealthy}, - }}, - }, - err: false, - }, - }), - - Entry("when machine phase is running", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineRunning}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeScaledDown}, - }}, - }, - err: false, - }, - }), - - Entry("when machine phase is terminating", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineTerminating}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeScaledDown}, - }}, - }, - err: false, - }, - }), - - Entry("when machine phase is terminating and condition already exists", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineTerminating}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Spec: corev1.NodeSpec{}, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeUnhealthy}, - }}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{ - "anno1": "anno1", - }, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeUnhealthy}, - }}, - }, - err: false, - }, - }), - - Entry("when phase changes and condition already exists", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineFailed}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - }, - Spec: corev1.NodeSpec{}, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeScaledDown}, - }}, - }, - }, - expect: expect{ - node: &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node-0", - Annotations: map[string]string{}, - }, - Status: corev1.NodeStatus{ - Conditions: []corev1.NodeCondition{ - {Type: NodeTerminationCondition, Status: corev1.ConditionTrue, Reason: NodeUnhealthy}, - }}, - }, - err: false, - }, - }), - - Entry("when node object does not exist", &data{ - setup: setup{ - machine: newMachine( - &machinev1.MachineTemplateSpec{}, - &machinev1.MachineStatus{ - Node: "test-node", - CurrentStatus: machinev1.CurrentStatus{Phase: MachineTerminating}, - }, - nil, nil, nil), - }, - action: action{ - node: &corev1.Node{}, - }, - expect: expect{ - node: &corev1.Node{}, - err: false, // we should not return error if node-object does not exist to ensure rest of the steps are then executed. - }, - }), - ) - - }) - -}) diff --git a/pkg/controller/machineclass_util.go b/pkg/controller/machineclass_util.go deleted file mode 100644 index 315c92f18..000000000 --- a/pkg/controller/machineclass_util.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "k8s.io/apimachinery/pkg/labels" -) - -func (c *controller) findMachineDeploymentsForClass(kind, name string) ([]*v1alpha1.MachineDeployment, error) { - machineDeployments, err := c.machineDeploymentLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.MachineDeployment - for _, machineDeployment := range machineDeployments { - if machineDeployment.Spec.Template.Spec.Class.Kind == kind && machineDeployment.Spec.Template.Spec.Class.Name == name { - filtered = append(filtered, machineDeployment) - } - } - return filtered, nil -} - -func (c *controller) findMachineSetsForClass(kind, name string) ([]*v1alpha1.MachineSet, error) { - machineSets, err := c.machineSetLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.MachineSet - for _, machineSet := range machineSets { - if machineSet.Spec.Template.Spec.Class.Kind == kind && machineSet.Spec.Template.Spec.Class.Name == name { - filtered = append(filtered, machineSet) - } - } - return filtered, nil -} - -func (c *controller) findMachinesForClass(kind, name string) ([]*v1alpha1.Machine, error) { - machines, err := c.machineLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.Machine - for _, machine := range machines { - if machine.Spec.Class.Kind == kind && machine.Spec.Class.Name == name { - filtered = append(filtered, machine) - } - } - return filtered, nil -} diff --git a/pkg/controller/machineset_test.go b/pkg/controller/machineset_test.go index 617a9055b..c282e7090 100644 --- a/pkg/controller/machineset_test.go +++ b/pkg/controller/machineset_test.go @@ -485,8 +485,6 @@ var _ = Describe("machineset", func() { Err := c.reconcileClusterMachineSet(Key) waitForCacheSync(stop, c) - machines, _ = c.controlMachineClient.Machines(testNamespace).List(context.TODO(), metav1.ListOptions{}) - //Expect(len(machines.Items)).To(Equal(int(testMachineSet.Spec.Replicas))) Expect(Err).Should(BeNil()) }) }) diff --git a/pkg/controller/node.go b/pkg/controller/node.go deleted file mode 100644 index 77fd14b8e..000000000 --- a/pkg/controller/node.go +++ /dev/null @@ -1,66 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - v1 "k8s.io/api/core/v1" - "k8s.io/klog/v2" - - "k8s.io/client-go/tools/cache" - - apierrors "k8s.io/apimachinery/pkg/api/errors" -) - -func (c *controller) nodeAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.nodeQueue.Add(key) -} - -func (c *controller) nodeUpdate(oldObj, newObj interface{}) { - c.nodeAdd(newObj) -} - -func (c *controller) nodeDelete(obj interface{}) { - node, ok := obj.(*v1.Node) - if node == nil || !ok { - return - } - -} - -// Not being used at the moment, saving it for a future use case. -func (c *controller) reconcileClusterNodeKey(key string) error { - node, err := c.nodeLister.Get(key) - if apierrors.IsNotFound(err) { - return nil - } - if err != nil { - klog.Errorf("ClusterNode %q: Unable to retrieve object from store: %v", key, err) - return err - } - - return c.reconcileClusterNode(node) -} - -func (c *controller) reconcileClusterNode(node *v1.Node) error { - return nil -} diff --git a/pkg/controller/openstackmachineclass.go b/pkg/controller/openstackmachineclass.go deleted file mode 100644 index 3f2ed5bf7..000000000 --- a/pkg/controller/openstackmachineclass.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// OpenStackMachineClassKind is used to identify the machineClassKind as OpenStack -const OpenStackMachineClassKind = "OpenStackMachineClass" - -func (c *controller) machineDeploymentToOpenStackMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == OpenStackMachineClassKind { - c.openStackMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToOpenStackMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == OpenStackMachineClassKind { - c.openStackMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToOpenStackMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", obj) - return - } - if machine.Spec.Class.Kind == OpenStackMachineClassKind { - c.openStackMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToOpenStackMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == OpenStackMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.openStackMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == OpenStackMachineClassKind { - c.openStackMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == OpenStackMachineClassKind { - c.openStackMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToOpenStackMachineClassDelete(obj interface{}) { - c.machineToOpenStackMachineClassAdd(obj) -} - -func (c *controller) openStackMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.openStackMachineClassQueue.Add(key) -} - -func (c *controller) openStackMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.OpenStackMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.OpenStackMachineClass) - if new == nil || !ok { - return - } - - c.openStackMachineClassAdd(newObj) -} - -func (c *controller) openStackMachineClassDelete(obj interface{}) { - c.openStackMachineClassAdd(obj) -} - -// reconcileClusterOpenStackMachineClassKey reconciles an OpenStackMachineClass due to controller resync -// or an event on the openStackMachineClass. -func (c *controller) reconcileClusterOpenStackMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.openStackMachineClassLister.OpenStackMachineClasses(c.namespace).Get(name) - if errors.IsNotFound(err) { - klog.V(4).Infof("%s %q: Not doing work because it has been deleted", OpenStackMachineClassKind, key) - return nil - } - if err != nil { - klog.V(4).Infof("%s %q: Unable to retrieve object from store: %v", OpenStackMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterOpenStackMachineClass(ctx, class) - if err != nil { - c.enqueueOpenStackMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Infuture to get ride of this logic - c.enqueueOpenStackMachineClassAfter(class, 10*time.Minute) - } - - return nil -} - -func (c *controller) reconcileClusterOpenStackMachineClass(ctx context.Context, class *v1alpha1.OpenStackMachineClass) error { - klog.V(4).Info("Start Reconciling OpenStackmachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling OpenStackmachineclass: ", class.Name) - - internalClass := &machine.OpenStackMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidateOpenStackMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of %s failed %s", OpenStackMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err := c.addOpenStackMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(OpenStackMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.OpenStackMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - return c.deleteOpenStackMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addOpenStackMachineClassFinalizers(ctx context.Context, class *v1alpha1.OpenStackMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateOpenStackMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteOpenStackMachineClassFinalizers(ctx context.Context, class *v1alpha1.OpenStackMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateOpenStackMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateOpenStackMachineClassFinalizers(ctx context.Context, class *v1alpha1.OpenStackMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.OpenStackMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.OpenStackMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating OpenStackMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the openstackmachineclass %q", class.Name) - return err -} - -func (c *controller) enqueueOpenStackMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.openStackMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/packetcloudmachineclass.go b/pkg/controller/packetcloudmachineclass.go deleted file mode 100644 index 77a8b90c4..000000000 --- a/pkg/controller/packetcloudmachineclass.go +++ /dev/null @@ -1,272 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "fmt" - "time" - - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - - "k8s.io/klog/v2" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "github.com/gardener/machine-controller-manager/pkg/apis/machine/validation" - "github.com/gardener/machine-controller-manager/pkg/util/provider/machineutils" -) - -// PacketMachineClassKind is used to identify the machineClassKind as Packet -const PacketMachineClassKind = "PacketMachineClass" - -func (c *controller) machineDeploymentToPacketMachineClassDelete(obj interface{}) { - machineDeployment, ok := obj.(*v1alpha1.MachineDeployment) - if machineDeployment == nil || !ok { - return - } - if machineDeployment.Spec.Template.Spec.Class.Kind == PacketMachineClassKind { - c.packetMachineClassQueue.Add(machineDeployment.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineSetToPacketMachineClassDelete(obj interface{}) { - machineSet, ok := obj.(*v1alpha1.MachineSet) - if machineSet == nil || !ok { - return - } - if machineSet.Spec.Template.Spec.Class.Kind == PacketMachineClassKind { - c.packetMachineClassQueue.Add(machineSet.Spec.Template.Spec.Class.Name) - } -} - -func (c *controller) machineToPacketMachineClassAdd(obj interface{}) { - machine, ok := obj.(*v1alpha1.Machine) - if machine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", obj) - return - } - if machine.Spec.Class.Kind == PacketMachineClassKind { - c.packetMachineClassQueue.Add(machine.Spec.Class.Name) - } -} - -func (c *controller) machineToPacketMachineClassUpdate(oldObj, newObj interface{}) { - oldMachine, ok := oldObj.(*v1alpha1.Machine) - if oldMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", oldObj) - return - } - newMachine, ok := newObj.(*v1alpha1.Machine) - if newMachine == nil || !ok { - klog.Warningf("Couldn't get machine from object: %+v", newObj) - return - } - - if oldMachine.Spec.Class.Kind == newMachine.Spec.Class.Kind { - if newMachine.Spec.Class.Kind == PacketMachineClassKind { - // Both old and new machine refer to the same machineClass object - // And the correct kind so enqueuing only one of them. - c.packetMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } else { - // If both are pointing to different machineClasses - // we might have to enqueue both. - if oldMachine.Spec.Class.Kind == PacketMachineClassKind { - c.packetMachineClassQueue.Add(oldMachine.Spec.Class.Name) - } - if newMachine.Spec.Class.Kind == PacketMachineClassKind { - c.packetMachineClassQueue.Add(newMachine.Spec.Class.Name) - } - } -} - -func (c *controller) machineToPacketMachineClassDelete(obj interface{}) { - c.machineToPacketMachineClassAdd(obj) -} - -func (c *controller) packetMachineClassAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.packetMachineClassQueue.Add(key) -} - -func (c *controller) packetMachineClassUpdate(oldObj, newObj interface{}) { - old, ok := oldObj.(*v1alpha1.PacketMachineClass) - if old == nil || !ok { - return - } - new, ok := newObj.(*v1alpha1.PacketMachineClass) - if new == nil || !ok { - return - } - - c.packetMachineClassAdd(newObj) -} - -func (c *controller) packetMachineClassDelete(obj interface{}) { - c.packetMachineClassAdd(obj) -} - -// reconcileClusterPacketMachineClassKey reconciles a PacketMachineClass due to controller resync -// or an event on the packetMachineClass. -func (c *controller) reconcileClusterPacketMachineClassKey(key string) error { - ctx := context.Background() - _, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } - - class, err := c.packetMachineClassLister.PacketMachineClasses(c.namespace).Get(name) - if errors.IsNotFound(err) { - klog.Infof("%s %q: Not doing work because it has been deleted", PacketMachineClassKind, key) - return nil - } - if err != nil { - klog.Infof("%s %q: Unable to retrieve object from store: %v", PacketMachineClassKind, key, err) - return err - } - - err = c.reconcileClusterPacketMachineClass(ctx, class) - if err != nil { - c.enqueuePacketMachineClassAfter(class, 10*time.Second) - } else { - // Re-enqueue periodically to avoid missing of events - // TODO: Infuture to get ride of this logic - c.enqueuePacketMachineClassAfter(class, 10*time.Minute) - } - - return nil -} - -func (c *controller) reconcileClusterPacketMachineClass(ctx context.Context, class *v1alpha1.PacketMachineClass) error { - klog.V(4).Info("Start Reconciling Packetmachineclass: ", class.Name) - defer klog.V(4).Info("Stop Reconciling Packetmachineclass: ", class.Name) - - internalClass := &machine.PacketMachineClass{} - err := c.internalExternalScheme.Convert(class, internalClass, nil) - if err != nil { - return err - } - - validationerr := validation.ValidatePacketMachineClass(internalClass) - if validationerr.ToAggregate() != nil && len(validationerr.ToAggregate().Errors()) > 0 { - klog.Errorf("Validation of %s failed %s", PacketMachineClassKind, validationerr.ToAggregate().Error()) - return nil - } - - // Add finalizer to avoid losing machineClass object - if class.DeletionTimestamp == nil { - err = c.addPacketMachineClassFinalizers(ctx, class) - if err != nil { - return err - } - } - - machines, err := c.findMachinesForClass(PacketMachineClassKind, class.Name) - if err != nil { - return err - } - - if class.DeletionTimestamp == nil { - // If deletion timestamp doesn't exist - _, annotationPresent := class.Annotations[machineutils.MigratedMachineClass] - - if c.deleteMigratedMachineClass && annotationPresent && len(machines) == 0 { - // If controller has deleteMigratedMachineClass flag set - // and the migratedMachineClass annotation is set - err = c.controlMachineClient.PacketMachineClasses(class.Namespace).Delete(ctx, class.Name, metav1.DeleteOptions{}) - if err != nil { - return err - } - return fmt.Errorf("Retry deletion as deletion timestamp is now set") - } - - return nil - } - - if len(machines) > 0 { - // machines are still referring the machine class, please wait before deletion - klog.V(3).Infof("Cannot remove finalizer on %s because still (%d) machines are referencing it", class.Name, len(machines)) - - for _, machine := range machines { - c.addMachine(machine) - } - - return fmt.Errorf("Retry as machine objects are still referring the machineclass") - } - - return c.deletePacketMachineClassFinalizers(ctx, class) -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addPacketMachineClassFinalizers(ctx context.Context, class *v1alpha1.PacketMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updatePacketMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deletePacketMachineClassFinalizers(ctx context.Context, class *v1alpha1.PacketMachineClass) error { - clone := class.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updatePacketMachineClassFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updatePacketMachineClassFinalizers(ctx context.Context, class *v1alpha1.PacketMachineClass, finalizers []string) error { - // Get the latest version of the class so that we can avoid conflicts - class, err := c.controlMachineClient.PacketMachineClasses(class.Namespace).Get(ctx, class.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := class.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlMachineClient.PacketMachineClasses(class.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - if err != nil { - klog.Warning("Updating PacketMachineClass failed, retrying. ", class.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the packetmachineclass %q", class.Name) - return err -} - -func (c *controller) enqueuePacketMachineClassAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.openStackMachineClassQueue.AddAfter(key, after) -} diff --git a/pkg/controller/secret.go b/pkg/controller/secret.go deleted file mode 100644 index 91924cd88..000000000 --- a/pkg/controller/secret.go +++ /dev/null @@ -1,340 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "context" - "time" - - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - corev1 "k8s.io/api/core/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/tools/cache" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog/v2" -) - -// reconcileClusterSecretKey reconciles an secret due to controller resync -// or an event on the secret -func (c *controller) reconcileClusterSecretKey(key string) error { - ctx := context.Background() - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - return err - } else if c.namespace != namespace { - // Secret exists outside of controller namespace - return nil - } - - secret, err := c.secretLister.Secrets(c.namespace).Get(name) - if errors.IsNotFound(err) { - klog.V(4).Infof("%q: Not doing work because it has been deleted", key) - return nil - } else if err != nil { - klog.V(4).Infof("%q: Unable to retrieve object from store: %v", key, err) - return err - } - - return c.reconcileClusterSecret(ctx, secret) -} - -// reconcileClusterSecret manipulates finalizers based on -// machineClass references -func (c *controller) reconcileClusterSecret(ctx context.Context, secret *corev1.Secret) error { - startTime := time.Now() - - klog.V(4).Infof("Start syncing %q", secret.Name) - defer func() { - c.enqueueSecretAfter(secret, 10*time.Minute) - klog.V(4).Infof("Finished syncing %q (%v)", secret.Name, time.Since(startTime)) - }() - - // Check if machineClasses are referring to this secret - exists, err := c.existsMachineClassForSecret(secret.Name) - if err != nil { - return err - } - - if exists { - // If one or more machineClasses refer this, add finalizer (if it doesn't exist) - err = c.addSecretFinalizers(ctx, secret) - if err != nil { - return err - } - } else { - if finalizers := sets.NewString(secret.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - // Finalizer doesn't exist, simply return nil - return nil - } - err = c.deleteSecretFinalizers(ctx, secret) - if err != nil { - return err - } - } - - return nil -} - -/* - SECTION - Manipulate Finalizers -*/ - -func (c *controller) addSecretFinalizers(ctx context.Context, secret *corev1.Secret) error { - clone := secret.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); !finalizers.Has(DeleteFinalizerName) { - finalizers.Insert(DeleteFinalizerName) - return c.updateSecretFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) deleteSecretFinalizers(ctx context.Context, secret *corev1.Secret) error { - clone := secret.DeepCopy() - - if finalizers := sets.NewString(clone.Finalizers...); finalizers.Has(DeleteFinalizerName) { - finalizers.Delete(DeleteFinalizerName) - return c.updateSecretFinalizers(ctx, clone, finalizers.List()) - } - return nil -} - -func (c *controller) updateSecretFinalizers(ctx context.Context, secret *corev1.Secret, finalizers []string) error { - // Get the latest version of the secret so that we can avoid conflicts - secret, err := c.controlCoreClient.CoreV1().Secrets(secret.Namespace).Get(ctx, secret.Name, metav1.GetOptions{}) - if err != nil { - return err - } - - clone := secret.DeepCopy() - clone.Finalizers = finalizers - _, err = c.controlCoreClient.CoreV1().Secrets(clone.Namespace).Update(ctx, clone, metav1.UpdateOptions{}) - - if err != nil { - klog.Warning("Updating secret finalizers failed, retrying", secret.Name, err) - return err - } - klog.V(3).Infof("Successfully added/removed finalizer on the secret %q", secret.Name) - return err -} - -/* - SECTION - Event handlers -*/ - -func (c *controller) secretAdd(obj interface{}) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) - if err != nil { - klog.Errorf("Couldn't get key for object %+v: %v", obj, err) - return - } - c.secretQueue.Add(key) -} - -func (c *controller) secretDelete(obj interface{}) { - c.secretAdd(obj) -} - -func (c *controller) enqueueSecretAfter(obj interface{}, after time.Duration) { - key, err := cache.MetaNamespaceKeyFunc(obj) - if err != nil { - return - } - c.secretQueue.AddAfter(key, after) -} - -func enqueueSecretForReferences(queue workqueue.RateLimitingInterface, secretRefs ...*corev1.SecretReference) { - for _, secretRef := range secretRefs { - if secretRef != nil { - queue.Add(secretRef.Namespace + "/" + secretRef.Name) - } - } -} - -func enqueueSecretForReferenceIfChanged(queue workqueue.RateLimitingInterface, oldSecretRef, newSecretRef *corev1.SecretReference) { - if !apiequality.Semantic.DeepEqual(oldSecretRef, newSecretRef) { - if oldSecretRef != nil { - queue.Add(oldSecretRef.Namespace + "/" + oldSecretRef.Name) - } - if newSecretRef != nil { - queue.Add(newSecretRef.Namespace + "/" + newSecretRef.Name) - } - } -} - -func (c *controller) openStackMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.OpenStackMachineClass) - if machineClass == nil || !ok { - return - } - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) openStackMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.OpenStackMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.OpenStackMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) openStackMachineClassToSecretDelete(obj interface{}) { - c.openStackMachineClassToSecretAdd(obj) -} - -func (c *controller) gcpMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.GCPMachineClass) - if machineClass == nil || !ok { - return - } - - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) gcpMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.GCPMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.GCPMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) gcpMachineClassToSecretDelete(obj interface{}) { - c.gcpMachineClassToSecretAdd(obj) -} - -func (c *controller) azureMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.AzureMachineClass) - if machineClass == nil || !ok { - return - } - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) azureMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.AzureMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.AzureMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) azureMachineClassToSecretDelete(obj interface{}) { - c.azureMachineClassToSecretAdd(obj) -} - -func (c *controller) alicloudMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.AlicloudMachineClass) - if machineClass == nil || !ok { - return - } - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) alicloudMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.AlicloudMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.AlicloudMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) alicloudMachineClassToSecretDelete(obj interface{}) { - c.alicloudMachineClassToSecretAdd(obj) -} - -func (c *controller) awsMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.AWSMachineClass) - if machineClass == nil || !ok { - return - } - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) awsMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.AWSMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.AWSMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) awsMachineClassToSecretDelete(obj interface{}) { - c.awsMachineClassToSecretAdd(obj) -} - -func (c *controller) packetMachineClassToSecretAdd(obj interface{}) { - machineClass, ok := obj.(*v1alpha1.PacketMachineClass) - if machineClass == nil || !ok { - return - } - enqueueSecretForReferences(c.secretQueue, machineClass.Spec.SecretRef, machineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) packetMachineClassToSecretUpdate(oldObj interface{}, newObj interface{}) { - oldMachineClass, ok := oldObj.(*v1alpha1.PacketMachineClass) - if oldMachineClass == nil || !ok { - return - } - newMachineClass, ok := newObj.(*v1alpha1.PacketMachineClass) - if newMachineClass == nil || !ok { - return - } - - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.SecretRef, newMachineClass.Spec.SecretRef) - enqueueSecretForReferenceIfChanged(c.secretQueue, oldMachineClass.Spec.CredentialsSecretRef, newMachineClass.Spec.CredentialsSecretRef) -} - -func (c *controller) packetMachineClassToSecretDelete(obj interface{}) { - c.packetMachineClassToSecretAdd(obj) -} diff --git a/pkg/controller/secret_test.go b/pkg/controller/secret_test.go deleted file mode 100644 index 11d514db6..000000000 --- a/pkg/controller/secret_test.go +++ /dev/null @@ -1,148 +0,0 @@ -/* -Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package controller - -import ( - "context" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" -) - -var _ = Describe("secret", func() { - - //TODO: This method has dependency on generic-machineclass. Implement later. - Describe("#reconcileClusterSecret", func() {}) - - Describe("#addSecretFinalizers", func() { - var ( - testSecret *corev1.Secret - ) - - BeforeEach(func() { - testSecret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "Secret-test", - Namespace: testNamespace, - }, - } - }) - - // Testcase: It should add finalizer on Secret. - It("should add finalizer on Secret.", func() { - stop := make(chan struct{}) - defer close(stop) - - objects := []runtime.Object{} - objects = append(objects, testSecret) - c, trackers := createController(stop, testNamespace, nil, objects, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - c.addSecretFinalizers(context.TODO(), testSecret) - - waitForCacheSync(stop, c) - expectedSecret, _ := c.controlCoreClient.CoreV1().Secrets(testSecret.Namespace).Get(context.TODO(), testSecret.Name, metav1.GetOptions{}) - - Expect(expectedSecret.Finalizers).To(HaveLen(1)) - Expect(expectedSecret.Finalizers).To(ContainElement(DeleteFinalizerName)) - }) - }) - - Describe("#deleteSecretFinalizers", func() { - var ( - testSecret *corev1.Secret - finalizers []string - ) - - BeforeEach(func() { - testSecret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "Secret-test", - Namespace: testNamespace, - }, - } - finalizers = []string{DeleteFinalizerName} - }) - - // Testcase: It should delete the finalizer from Secret. - It("should delete the finalizer from Secret.", func() { - stop := make(chan struct{}) - defer close(stop) - - objects := []runtime.Object{} - objects = append(objects, testSecret) - c, trackers := createController(stop, testNamespace, nil, objects, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - testSecret, _ := c.controlCoreClient.CoreV1().Secrets(testSecret.Namespace).Get(context.TODO(), testSecret.Name, metav1.GetOptions{}) - - testSecret.Finalizers = finalizers - Expect(testSecret.Finalizers).Should(Not(BeEmpty())) - - c.deleteSecretFinalizers(context.TODO(), testSecret) - - waitForCacheSync(stop, c) - - expectedSecret, _ := c.controlCoreClient.CoreV1().Secrets(testSecret.Namespace).Get(context.TODO(), testSecret.Name, metav1.GetOptions{}) - - Expect(expectedSecret.Finalizers).Should(HaveLen(0)) - }) - }) - - Describe("#updateSecretFinalizers", func() { - var ( - testSecret *corev1.Secret - finalizers []string - ) - - BeforeEach(func() { - testSecret = &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "Secret-test", - Namespace: testNamespace, - }, - } - - finalizers = []string{"finalizer1", "finalizer2"} - }) - - // Testcase: It should update the finalizer on Secret. - It("should update the finalizer on Secret.", func() { - stop := make(chan struct{}) - defer close(stop) - - objects := []runtime.Object{} - objects = append(objects, testSecret) - c, trackers := createController(stop, testNamespace, nil, objects, nil) - defer trackers.Stop() - waitForCacheSync(stop, c) - - c.updateSecretFinalizers(context.TODO(), testSecret, finalizers) - - waitForCacheSync(stop, c) - - testSecret, _ := c.controlCoreClient.CoreV1().Secrets(testSecret.Namespace).Get(context.TODO(), testSecret.Name, metav1.GetOptions{}) - - Expect(testSecret.Finalizers).To(Equal(finalizers)) - }) - }) -}) diff --git a/pkg/controller/secret_util.go b/pkg/controller/secret_util.go deleted file mode 100644 index 60e884cc9..000000000 --- a/pkg/controller/secret_util.go +++ /dev/null @@ -1,170 +0,0 @@ -/* -Copyright (c) 2017 SAP SE or an SAP affiliate company. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package controller is used to provide the core functionalities of machine-controller-manager -package controller - -import ( - "github.com/gardener/machine-controller-manager/pkg/apis/machine/v1alpha1" - "k8s.io/apimachinery/pkg/labels" -) - -// existsMachineClassForSecret checks for any machineClass -// referring to the passed secret object -func (c *controller) existsMachineClassForSecret(name string) (bool, error) { - openStackMachineClasses, err := c.findOpenStackMachineClassForSecret(name) - if err != nil { - return false, err - } - - gcpMachineClasses, err := c.findGCPMachineClassForSecret(name) - if err != nil { - return false, err - } - - azureMachineClasses, err := c.findAzureMachineClassForSecret(name) - if err != nil { - return false, err - } - - awsMachineClasses, err := c.findAWSMachineClassForSecret(name) - if err != nil { - return false, err - } - - alicloudMachineClasses, err := c.findAlicloudMachineClassForSecret(name) - if err != nil { - return false, err - } - - packetMachineClasses, err := c.findPacketMachineClassForSecret(name) - if err != nil { - return false, err - } - - if len(openStackMachineClasses) == 0 && - len(gcpMachineClasses) == 0 && - len(azureMachineClasses) == 0 && - len(packetMachineClasses) == 0 && - len(alicloudMachineClasses) == 0 && - len(awsMachineClasses) == 0 { - return false, nil - } - - return true, nil -} - -// findOpenStackMachineClassForSecret returns the set of -// openStackMachineClasses referring to the passed secret -func (c *controller) findOpenStackMachineClassForSecret(name string) ([]*v1alpha1.OpenStackMachineClass, error) { - machineClasses, err := c.openStackMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.OpenStackMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -} - -// findGCPClassForSecret returns the set of -// GCPMachineClasses referring to the passed secret -func (c *controller) findGCPMachineClassForSecret(name string) ([]*v1alpha1.GCPMachineClass, error) { - machineClasses, err := c.gcpMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.GCPMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -} - -// findAzureClassForSecret returns the set of -// AzureMachineClasses referring to the passed secret -func (c *controller) findAzureMachineClassForSecret(name string) ([]*v1alpha1.AzureMachineClass, error) { - machineClasses, err := c.azureMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.AzureMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -} - -// findAlicloudClassForSecret returns the set of -// AlicloudMachineClasses referring to the passed secret -func (c *controller) findAlicloudMachineClassForSecret(name string) ([]*v1alpha1.AlicloudMachineClass, error) { - machineClasses, err := c.alicloudMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.AlicloudMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -} - -// findAWSClassForSecret returns the set of -// AWSMachineClasses referring to the passed secret -func (c *controller) findAWSMachineClassForSecret(name string) ([]*v1alpha1.AWSMachineClass, error) { - machineClasses, err := c.awsMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.AWSMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -} - -// findPacketClassForSecret returns the set of -// PacketMachineClasses referring to the passed secret -func (c *controller) findPacketMachineClassForSecret(name string) ([]*v1alpha1.PacketMachineClass, error) { - machineClasses, err := c.packetMachineClassLister.List(labels.Everything()) - if err != nil { - return nil, err - } - var filtered []*v1alpha1.PacketMachineClass - for _, machineClass := range machineClasses { - if (machineClass.Spec.SecretRef != nil && machineClass.Spec.SecretRef.Name == name) || - (machineClass.Spec.CredentialsSecretRef != nil && machineClass.Spec.CredentialsSecretRef.Name == name) { - filtered = append(filtered, machineClass) - } - } - return filtered, nil -}