Merge branch 'master' into feature/maxPods

Signed-off-by: Jonah Back <[email protected]>

.github/workflows/push.yml

@@ -4,9 +4,7 @@ on:
   push:
     branches: [ master ]
     tags:
-    - *
-  pull_request:
-    branches: [ master ]
+    - '*'
 
 jobs:
   build:

api/v1alpha1/instancegroup_types.go

@@ -314,6 +314,7 @@ type InstanceGroupStatus struct {
 	ActiveScalingGroupName        string                   `json:"activeScalingGroupName,omitempty"`
 	NodesArn                      string                   `json:"nodesInstanceRoleArn,omitempty"`
 	StrategyResourceName          string                   `json:"strategyResourceName,omitempty"`
+	StrategyResourceNamespace     string                   `json:"strategyResourceNamespace,omitempty"`
 	UsingSpotRecommendation       bool                     `json:"usingSpotRecommendation,omitempty"`
 	Lifecycle                     string                   `json:"lifecycle,omitempty"`
 	ConfigHash                    string                   `json:"configMD5,omitempty"`
@@ -907,10 +908,18 @@ func (status *InstanceGroupStatus) GetStrategyResourceName() string {
 	return status.StrategyResourceName
 }
 
+func (status *InstanceGroupStatus) GetStrategyResourceNamespace() string {
+	return status.StrategyResourceNamespace
+}
+
 func (status *InstanceGroupStatus) SetStrategyResourceName(name string) {
 	status.StrategyResourceName = name
 }
 
+func (status *InstanceGroupStatus) SetStrategyResourceNamespace(namespace string) {
+	status.StrategyResourceNamespace = namespace
+}
+
 func (status *InstanceGroupStatus) GetCurrentMin() int {
 	return status.CurrentMin
 }

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -425,6 +425,8 @@ spec:
               type: string
             strategyResourceName:
               type: string
+            strategyResourceNamespace:
+              type: string
             usingSpotRecommendation:
               type: boolean
           type: object

controllers/providers/kubernetes/crd.go

@@ -25,6 +25,7 @@ import (
 	kerr "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 )
 
@@ -81,6 +82,7 @@ func ProcessCRDStrategy(kube dynamic.Interface, instanceGroup *v1alpha1.Instance
 		return false, errors.Errorf("custom resource definition '%v' is missing, could not upgrade", crdFullName)
 	}
 	status.SetStrategyResourceName(customResource.GetName())
+	status.SetStrategyResourceNamespace(customResource.GetNamespace())
 
 	policy := strategy.GetConcurrencyPolicy()
 
@@ -199,6 +201,75 @@ func NormalizeName(customResource *unstructured.Unstructured, id string) {
 	}
 }
 
+func ResourceGVR(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup) (schema.GroupVersionResource, error) {
+	var (
+		strategy = instanceGroup.GetUpgradeStrategy().GetCRDType()
+	)
+
+	renderParams := struct {
+		InstanceGroup *v1alpha1.InstanceGroup
+	}{
+		InstanceGroup: instanceGroup,
+	}
+
+	templatedCustomResource, err := RenderCustomResource(strategy.GetSpec(), renderParams)
+	if err != nil {
+		return schema.GroupVersionResource{}, errors.Wrap(err, "failed to render custom resource templating")
+	}
+
+	customResource, err := ParseCustomResourceYaml(templatedCustomResource)
+	if err != nil {
+		return schema.GroupVersionResource{}, errors.Wrap(err, "failed to parse custom resource yaml")
+	}
+
+	return GetGVR(customResource, strategy.GetCRDName()), nil
+}
+
+func IsResourceActive(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup) bool {
+	var (
+		strategy = instanceGroup.GetUpgradeStrategy().GetCRDType()
+		status   = instanceGroup.GetStatus()
+	)
+
+	if strategy == nil {
+		return false
+	}
+
+	var (
+		statusJSONPath  = strategy.GetStatusJSONPath()
+		completedStatus = strategy.GetStatusSuccessString()
+		errorStatus     = strategy.GetStatusFailureString()
+	)
+
+	name := status.GetStrategyResourceName()
+	namespace := status.GetStrategyResourceNamespace()
+
+	if common.StringEmpty(name) || common.StringEmpty(namespace) {
+		return false
+	}
+
+	gvr, err := ResourceGVR(kube, instanceGroup)
+	if err != nil {
+		log.Error(err, "failed to get resource gvr")
+		return false
+	}
+
+	r, err := kube.Resource(gvr).Namespace(namespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		if kerr.IsNotFound(err) {
+			return false
+		}
+		log.Error(err, "failed to get upgrade resource")
+		return false
+	}
+
+	if IsPathValue(*r, statusJSONPath, completedStatus) || IsPathValue(*r, statusJSONPath, errorStatus) {
+		return false
+	}
+
+	return true
+}
+
 func GetResources(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup, resource *unstructured.Unstructured) ([]*unstructured.Unstructured, []*unstructured.Unstructured, error) {
 	var (
 		status            = instanceGroup.GetStatus()
@@ -223,17 +294,13 @@ func GetResources(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup,
 			continue
 		}
 
-		val, err := GetUnstructuredPath(&r, statusJSONPath)
-		if err != nil {
-			return inactiveResources, activeResources, err
-		}
-
-		if strings.EqualFold(val, completedStatus) || strings.EqualFold(val, errorStatus) {
+		if IsPathValue(r, statusJSONPath, completedStatus) || IsPathValue(r, statusJSONPath, errorStatus) {
 			// if resource is not completed and not failed, it must be still active
 			inactiveResources = append(inactiveResources, &r)
 		} else {
 			activeResources = append(activeResources, &r)
 		}
+
 	}
 
 	return inactiveResources, activeResources, nil

controllers/providers/kubernetes/utils.go

@@ -263,3 +263,17 @@ func IsStorageError(err error) bool {
 	}
 	return false
 }
+
+func IsPathValue(resource unstructured.Unstructured, path, value string) bool {
+	val, err := GetUnstructuredPath(&resource, path)
+	if err != nil {
+		log.Error(err, "failed to get unstructured path from resource", "path", path)
+		return false
+	}
+
+	if strings.EqualFold(val, value) {
+		return true
+	}
+
+	return false
+}

controllers/provisioners/eks/update.go

@@ -27,6 +27,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/keikoproj/instance-manager/api/v1alpha1"
 	"github.com/keikoproj/instance-manager/controllers/common"
+	kubeprovider "github.com/keikoproj/instance-manager/controllers/providers/kubernetes"
 	"github.com/keikoproj/instance-manager/controllers/provisioners/eks/scaling"
 )
 
@@ -87,11 +88,17 @@ func (ctx *EksInstanceGroupContext) Update() error {
 		rotationNeeded = true
 	}
 
+	if kubeprovider.IsResourceActive(ctx.KubernetesClient.KubeDynamic, instanceGroup) {
+		ctx.Log.Info("upgrade resource is still active", "instancegroup", instanceGroup.GetName(), "scalingconfig", config.Name)
+		rotationNeeded = true
+	}
+
 	// update scaling group
 	err = ctx.UpdateScalingGroup(config.Name)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
-			if aerr.Code() != autoscaling.ErrCodeScalingActivityInProgressFault {
+			if aerr.Code() == autoscaling.ErrCodeScalingActivityInProgressFault {
+				ctx.Log.Info("cannot update scaling group due to autoscaling activity in progress", "instancegroup", instanceGroup.GetName())
 				return nil
 			}
 		}

docs/INSTALL.md

@@ -62,6 +62,7 @@ ec2:CreateLaunchTemplateVersion
 ec2:ModifyLaunchTemplate
 ec2:DeleteLaunchTemplate
 ec2:DeleteLaunchTemplateVersions
+ec2:RunInstances
 autoscaling:CreateOrUpdateTags
 autoscaling:DeleteTags
 autoscaling:SuspendProcesses