AL2 Support for AWS Warm Pools

api/v1alpha1/instancegroup_types.go

@@ -190,9 +190,15 @@ type BootstrapOptions struct {
 	MaxPods int64 `json:"maxPods,omitempty"`
 }
 
+type WarmPoolSpec struct {
+	MaxSize int64 `json:"maxSize,omitempty"`
+	MinSize int64 `json:"minSize,omitempty"`
+}
+
 type EKSSpec struct {
 	MaxSize          int64                    `json:"maxSize,omitempty"`
 	MinSize          int64                    `json:"minSize,omitempty"`
+	WarmPool         *WarmPoolSpec            `json:"warmPool,omitempty"`
 	Type             ScalingConfigurationType `json:"type,omitempty"`
 	EKSConfiguration *EKSConfiguration        `json:"configuration"`
 }
@@ -374,7 +380,11 @@ func (ig *InstanceGroup) SetUpgradeStrategy(strategy AwsUpgradeStrategy) {
 }
 
 func (s *EKSSpec) Validate() error {
-	if s.EKSConfiguration == nil {
+	var (
+		configuration = s.EKSConfiguration
+		configType    = s.Type
+	)
+	if configuration == nil {
 		return errors.Errorf("validation failed, 'configuration' is a required field")
 	}
 
@@ -396,6 +406,29 @@ func (s *EKSSpec) Validate() error {
 		}
 	}
 
+	for _, v := range configuration.Volumes {
+		if configType == LaunchConfiguration {
+			if !common.ContainsEqualFold(awsprovider.ConfigurationAllowedVolumeTypes, v.Type) {
+				return errors.Errorf("validation failed, volume type '%v' is unsupported", v.Type)
+			}
+		}
+
+		if configType == LaunchTemplate {
+			if !common.ContainsEqualFold(awsprovider.TemplateAllowedVolumeTypes, v.Type) {
+				return errors.Errorf("validation failed, volume type '%v' is unsupported", v.Type)
+			}
+		}
+	}
+
+	if s.HasWarmPool() {
+		if configuration.MixedInstancesPolicy != nil {
+			return errors.Errorf("validation failed, cannot use warmPool with MixedInstancesPolicy")
+		}
+		if !common.StringEmpty(configuration.SpotPrice) {
+			return errors.Errorf("validation failed, cannot use warmPool with SpotPrice")
+		}
+	}
+
 	return nil
 }
 
@@ -413,7 +446,14 @@ func (s *EKSSpec) IsLaunchConfiguration() bool {
 	return false
 }
 
-func (c *EKSConfiguration) Validate(scalingConfigurationType ScalingConfigurationType) error {
+func (s *EKSSpec) HasWarmPool() bool {
+	if s.WarmPool != nil {
+		return true
+	}
+	return false
+}
+
+func (c *EKSConfiguration) Validate() error {
 	if common.StringEmpty(c.EksClusterName) {
 		return errors.Errorf("validation failed, 'clusterName' is a required parameter")
 	}
@@ -490,13 +530,6 @@ func (c *EKSConfiguration) Validate(scalingConfigurationType ScalingConfiguratio
 	}
 
 	for _, v := range c.Volumes {
-		if scalingConfigurationType == LaunchConfiguration && !common.ContainsEqualFold(awsprovider.ConfigurationAllowedVolumeTypes, v.Type) {
-			return errors.Errorf("validation failed, volume type '%v' is unsupported", v.Type)
-		}
-
-		if scalingConfigurationType == LaunchTemplate && !common.ContainsEqualFold(awsprovider.TemplateAllowedVolumeTypes, v.Type) {
-			return errors.Errorf("validation failed, volume type '%v' is unsupported", v.Type)
-		}
 
 		if v.Iops != 0 && !common.ContainsEqualFold(awsprovider.AllowedVolumeTypesWithProvisionedIOPS, v.Type) {
 			log.Info("cannot apply IOPS configuration for volumeType, only types ['io1','io2','gp3'] supported", "volumeType", v.Type)
@@ -648,7 +681,7 @@ func (ig *InstanceGroup) Validate() error {
 			return err
 		}
 
-		if err := config.Validate(spec.Type); err != nil {
+		if err := config.Validate(); err != nil {
 			return err
 		}
 	}
@@ -804,6 +837,12 @@ func (spec *EKSSpec) GetMaxSize() int64 {
 func (spec *EKSSpec) GetMinSize() int64 {
 	return spec.MinSize
 }
+func (spec *WarmPoolSpec) GetMaxSize() int64 {
+	return spec.MaxSize
+}
+func (spec *WarmPoolSpec) GetMinSize() int64 {
+	return spec.MinSize
+}
 func (spec *EKSSpec) GetType() ScalingConfigurationType {
 	return spec.Type
 }

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -268,6 +268,15 @@ spec:
                     type: integer
                   type:
                     type: string
+                  warmPool:
+                    properties:
+                      maxSize:
+                        format: int64
+                        type: integer
+                      minSize:
+                        format: int64
+                        type: integer
+                    type: object
                 required:
                 - configuration
                 type: object

controllers/providers/aws/aws.go

@@ -46,6 +46,7 @@ var (
 
 const (
 	CacheDefaultTTL                   time.Duration = 0 * time.Second
+	DescribeWarmPoolTTL               time.Duration = 60 * time.Second
 	DescribeAutoScalingGroupsTTL      time.Duration = 60 * time.Second
 	DescribeLaunchConfigurationsTTL   time.Duration = 60 * time.Second
 	ListAttachedRolePoliciesTTL       time.Duration = 60 * time.Second
@@ -229,6 +230,39 @@ func (w *AwsWorker) CreateLifecycleHook(input *autoscaling.PutLifecycleHookInput
 	return nil
 }
 
+func (w *AwsWorker) DescribeWarmPool(asgName string) (*autoscaling.DescribeWarmPoolOutput, error) {
+	describeWarmPoolOutput, err := w.AsgClient.DescribeWarmPool(&autoscaling.DescribeWarmPoolInput{
+		AutoScalingGroupName: aws.String(asgName),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return describeWarmPoolOutput, nil
+}
+
+func (w *AwsWorker) UpdateWarmPool(asgName string, min, max int64) error {
+	_, err := w.AsgClient.PutWarmPool(&autoscaling.PutWarmPoolInput{
+		AutoScalingGroupName:     aws.String(asgName),
+		MaxGroupPreparedCapacity: aws.Int64(max),
+		MinSize:                  aws.Int64(min),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *AwsWorker) DeleteWarmPool(asgName string) error {
+	_, err := w.AsgClient.DeleteWarmPool(&autoscaling.DeleteWarmPoolInput{
+		AutoScalingGroupName: aws.String(asgName),
+		ForceDelete:          aws.Bool(true),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (w *AwsWorker) DeleteLifecycleHook(asgName, hookName string) error {
 	_, err := w.AsgClient.DeleteLifecycleHook(&autoscaling.DeleteLifecycleHookInput{
 		AutoScalingGroupName: aws.String(asgName),
@@ -1060,6 +1094,7 @@ func GetAwsAsgClient(region string, cacheCfg *cache.Config, maxRetries int, coll
 
 	cache.AddCaching(sess, cacheCfg)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeAutoScalingGroups", DescribeAutoScalingGroupsTTL)
+	cacheCfg.SetCacheTTL("autoscaling", "DescribeWarmPool", DescribeWarmPoolTTL)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeLaunchConfigurations", DescribeLaunchConfigurationsTTL)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeLifecycleHooks", DescribeLifecycleHooksTTL)
 	sess.Handlers.Complete.PushFront(func(r *request.Request) {

controllers/providers/aws/predicates.go

@@ -98,3 +98,10 @@ func IsUsingMixedInstances(group *autoscaling.Group) bool {
 	}
 	return false
 }
+
+func IsUsingWarmPool(group *autoscaling.Group) bool {
+	if group.WarmPoolConfiguration != nil {
+		return true
+	}
+	return false
+}

controllers/provisioners/config.go

@@ -297,7 +297,7 @@ func (c *ProvisionerConfiguration) setSharedFields(obj map[string]interface{}) e
 			if conditionalValue != nil {
 				if isConflict(conditionalValue, defaultVal) {
 					//Merge conditional into default, with conditional overriding any conflicting values.
-					merge := Merge(conditionalValue,defaultVal, pathStr, false)
+					merge := Merge(conditionalValue, defaultVal, pathStr, false)
 					if err := common.SetFieldValue(pathStr, obj, merge); err != nil {
 						return errors.Wrap(err, "failed to merge field")
 					}

controllers/provisioners/eks/cloud_test.go

@@ -477,5 +477,5 @@ func TestLaunchConfigDeletion(t *testing.T) {
 
 	err := ctx.CloudDiscovery()
 	g.Expect(err).NotTo(gomega.HaveOccurred())
-	g.Expect(asgMock.DeleteLaunchConfigurationCallCount).To(gomega.Equal(2))
+	g.Expect(asgMock.DeleteLaunchConfigurationCallCount).To(gomega.Equal(uint(2)))
 }

controllers/provisioners/eks/create_test.go

@@ -132,7 +132,7 @@ func TestCreateLaunchTemplatePositive(t *testing.T) {
 
 	g.Expect(ig.GetStatus().GetActiveLaunchTemplateName()).To(gomega.HavePrefix(prefix))
 	g.Expect(ctx.GetState()).To(gomega.Equal(v1alpha1.ReconcileModified))
-	g.Expect(ec2Mock.CreateLaunchTemplateCallCount).To(gomega.Equal(1))
+	g.Expect(ec2Mock.CreateLaunchTemplateCallCount).To(gomega.Equal(uint(1)))
 }
 
 func TestCreateScalingGroupPositive(t *testing.T) {

controllers/provisioners/eks/eks.go

@@ -53,8 +53,9 @@ var (
 	InstanceMgrLifecycleLabel = "instancemgr.keikoproj.io/lifecycle"
 	InstanceMgrImageLabel     = "instancemgr.keikoproj.io/image"
 
-	DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
-	CNIManagedPolicy       = "AmazonEKS_CNI_Policy"
+	DefaultManagedPolicies    = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
+	CNIManagedPolicy          = "AmazonEKS_CNI_Policy"
+	AutoscalingReadOnlyPolicy = "AutoScalingReadOnlyAccess"
 )
 
 // New constructs a new instance group provisioner of EKS type

controllers/provisioners/eks/eks_test.go

@@ -94,6 +94,21 @@ func MockEksCluster(version string) *eks.Cluster {
 	}
 }
 
+func MockWarmPoolSpec(maxSize, minSize int64) *v1alpha1.WarmPoolSpec {
+	return &v1alpha1.WarmPoolSpec{
+		MaxSize: maxSize,
+		MinSize: minSize,
+	}
+}
+
+func MockWarmPool(maxSize, minSize int64, status string) *autoscaling.WarmPoolConfiguration {
+	return &autoscaling.WarmPoolConfiguration{
+		MaxGroupPreparedCapacity: aws.Int64(maxSize),
+		MinSize:                  aws.Int64(minSize),
+		Status:                   aws.String(status),
+	}
+}
+
 func MockKubernetesClientSet() kubeprovider.KubernetesClientSet {
 	return kubeprovider.KubernetesClientSet{
 		Kubernetes:  fake.NewSimpleClientset(),
@@ -483,13 +498,20 @@ type MockAutoScalingClient struct {
 	DescribeLifecycleHooksErr              error
 	PutLifecycleHookErr                    error
 	DeleteLifecycleHookErr                 error
-	DeleteLaunchConfigurationCallCount     int
-	PutLifecycleHookCallCount              int
-	DeleteLifecycleHookCallCount           int
+	DescribeWarmPoolErr                    error
+	DeleteWarmPoolErr                      error
+	PutWarmPoolErr                         error
+	DeleteLaunchConfigurationCallCount     uint
+	PutLifecycleHookCallCount              uint
+	DeleteLifecycleHookCallCount           uint
+	PutWarmPoolCallCount                   uint
+	DeleteWarmPoolCallCount                uint
+	DescribeWarmPoolCallCount              uint
 	LaunchConfiguration                    *autoscaling.LaunchConfiguration
 	LaunchConfigurations                   []*autoscaling.LaunchConfiguration
 	AutoScalingGroup                       *autoscaling.Group
 	AutoScalingGroups                      []*autoscaling.Group
+	WarmPoolInstances                      []*autoscaling.Instance
 	LifecycleHooks                         []*autoscaling.LifecycleHook
 }
 
@@ -582,14 +604,29 @@ func (a *MockAutoScalingClient) PutLifecycleHook(input *autoscaling.PutLifecycle
 	return &autoscaling.PutLifecycleHookOutput{}, a.PutLifecycleHookErr
 }
 
+func (a *MockAutoScalingClient) DescribeWarmPool(input *autoscaling.DescribeWarmPoolInput) (*autoscaling.DescribeWarmPoolOutput, error) {
+	a.DescribeWarmPoolCallCount++
+	return &autoscaling.DescribeWarmPoolOutput{Instances: a.WarmPoolInstances}, a.DescribeWarmPoolErr
+}
+
+func (a *MockAutoScalingClient) DeleteWarmPool(input *autoscaling.DeleteWarmPoolInput) (*autoscaling.DeleteWarmPoolOutput, error) {
+	a.DeleteWarmPoolCallCount++
+	return &autoscaling.DeleteWarmPoolOutput{}, a.DeleteWarmPoolErr
+}
+
+func (a *MockAutoScalingClient) PutWarmPool(input *autoscaling.PutWarmPoolInput) (*autoscaling.PutWarmPoolOutput, error) {
+	a.PutWarmPoolCallCount++
+	return &autoscaling.PutWarmPoolOutput{}, a.PutWarmPoolErr
+}
+
 type MockEc2Client struct {
 	ec2iface.EC2API
 	DescribeSubnetsErr                   error
 	DescribeSecurityGroupsErr            error
-	CreateLaunchTemplateCallCount        int
-	CreateLaunchTemplateVersionCallCount int
-	ModifyLaunchTemplateCallCount        int
-	DeleteLaunchTemplateCallCount        int
+	CreateLaunchTemplateCallCount        uint
+	CreateLaunchTemplateVersionCallCount uint
+	ModifyLaunchTemplateCallCount        uint
+	DeleteLaunchTemplateCallCount        uint
 	Subnets                              []*ec2.Subnet
 	SecurityGroups                       []*ec2.SecurityGroup
 	LaunchTemplates                      []*ec2.LaunchTemplate
@@ -718,9 +755,9 @@ type MockIamClient struct {
 	AddRoleToInstanceProfileErr       error
 	RemoveRoleFromInstanceProfileErr  error
 	AttachRolePolicyErr               error
-	AttachRolePolicyCallCount         int
+	AttachRolePolicyCallCount         uint
 	DetachRolePolicyErr               error
-	DetachRolePolicyCallCount         int
+	DetachRolePolicyCallCount         uint
 	WaitUntilInstanceProfileExistsErr error
 	ListAttachedRolePoliciesErr       error
 	Role                              *iam.Role