AL2 Support for AWS Warm Pools

api/v1alpha1/instancegroup_types.go

@@ -190,9 +190,15 @@ type BootstrapOptions struct {
 	MaxPods int64 `json:"maxPods,omitempty"`
 }
 
+type WarmPoolSpec struct {
+	MaxSize int64 `json:"maxSize,omitempty"`
+	MinSize int64 `json:"minSize,omitempty"`
+}
+
 type EKSSpec struct {
 	MaxSize          int64                    `json:"maxSize,omitempty"`
 	MinSize          int64                    `json:"minSize,omitempty"`
+	WarmPool         *WarmPoolSpec            `json:"warmPool,omitempty"`
 	Type             ScalingConfigurationType `json:"type,omitempty"`
 	EKSConfiguration *EKSConfiguration        `json:"configuration"`
 }
@@ -413,7 +419,14 @@ func (s *EKSSpec) IsLaunchConfiguration() bool {
 	return false
 }
 
-func (c *EKSConfiguration) Validate(scalingConfigurationType ScalingConfigurationType) error {
+func (s *EKSSpec) HasWarmPool() bool {
+	if s.WarmPool != nil {
+		return true
+	}
+	return false
+}
+
+func (c *EKSConfiguration) Validate(scalingConfigurationType ScalingConfigurationType, hasWarmPool bool) error {
 	if common.StringEmpty(c.EksClusterName) {
 		return errors.Errorf("validation failed, 'clusterName' is a required parameter")
 	}
@@ -523,11 +536,20 @@ func (c *EKSConfiguration) Validate(scalingConfigurationType ScalingConfiguratio
 	}
 
 	if c.MixedInstancesPolicy != nil {
+		if hasWarmPool {
+			return errors.Errorf("validation failed, cannot use warmPool with MixedInstancesPolicy")
+		}
 		if err := c.MixedInstancesPolicy.Validate(); err != nil {
 			return err
 		}
 	}
 
+	if !common.StringEmpty(c.SpotPrice) {
+		if hasWarmPool {
+			return errors.Errorf("validation failed, cannot use warmPool with SpotPrice")
+		}
+	}
+
 	for i, v := range c.LicenseSpecifications {
 		if !strings.HasPrefix(v, awsprovider.ARNPrefix) {
 			return errors.Errorf("validation failed, 'LicenseSpecifications[%d]' must be a valid IAM role ARN", i)
@@ -648,7 +670,7 @@ func (ig *InstanceGroup) Validate() error {
 			return err
 		}
 
-		if err := config.Validate(spec.Type); err != nil {
+		if err := config.Validate(spec.Type, spec.HasWarmPool()); err != nil {
 			return err
 		}
 	}
@@ -804,6 +826,12 @@ func (spec *EKSSpec) GetMaxSize() int64 {
 func (spec *EKSSpec) GetMinSize() int64 {
 	return spec.MinSize
 }
+func (spec *WarmPoolSpec) GetMaxSize() int64 {
+	return spec.MaxSize
+}
+func (spec *WarmPoolSpec) GetMinSize() int64 {
+	return spec.MinSize
+}
 func (spec *EKSSpec) GetType() ScalingConfigurationType {
 	return spec.Type
 }

config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml

@@ -268,6 +268,15 @@ spec:
                     type: integer
                   type:
                     type: string
+                  warmPool:
+                    properties:
+                      maxSize:
+                        format: int64
+                        type: integer
+                      minSize:
+                        format: int64
+                        type: integer
+                    type: object
                 required:
                 - configuration
                 type: object

controllers/providers/aws/aws.go

@@ -46,6 +46,7 @@ var (
 
 const (
 	CacheDefaultTTL                   time.Duration = 0 * time.Second
+	DescribeWarmPoolTTL               time.Duration = 60 * time.Second
 	DescribeAutoScalingGroupsTTL      time.Duration = 60 * time.Second
 	DescribeLaunchConfigurationsTTL   time.Duration = 60 * time.Second
 	ListAttachedRolePoliciesTTL       time.Duration = 60 * time.Second
@@ -229,6 +230,39 @@ func (w *AwsWorker) CreateLifecycleHook(input *autoscaling.PutLifecycleHookInput
 	return nil
 }
 
+func (w *AwsWorker) DescribeWarmPool(asgName string) (*autoscaling.DescribeWarmPoolOutput, error) {
+	describeWarmPoolOutput, err := w.AsgClient.DescribeWarmPool(&autoscaling.DescribeWarmPoolInput{
+		AutoScalingGroupName: aws.String(asgName),
+	})
+	if err != nil {
+		return nil, err
+	}
+	return describeWarmPoolOutput, nil
+}
+
+func (w *AwsWorker) UpdateWarmPool(asgName string, min, max int64) error {
+	_, err := w.AsgClient.PutWarmPool(&autoscaling.PutWarmPoolInput{
+		AutoScalingGroupName:     aws.String(asgName),
+		MaxGroupPreparedCapacity: aws.Int64(max),
+		MinSize:                  aws.Int64(min),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (w *AwsWorker) DeleteWarmPool(asgName string) error {
+	_, err := w.AsgClient.DeleteWarmPool(&autoscaling.DeleteWarmPoolInput{
+		AutoScalingGroupName: aws.String(asgName),
+		ForceDelete:          aws.Bool(true),
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (w *AwsWorker) DeleteLifecycleHook(asgName, hookName string) error {
 	_, err := w.AsgClient.DeleteLifecycleHook(&autoscaling.DeleteLifecycleHookInput{
 		AutoScalingGroupName: aws.String(asgName),
@@ -1060,6 +1094,7 @@ func GetAwsAsgClient(region string, cacheCfg *cache.Config, maxRetries int, coll
 
 	cache.AddCaching(sess, cacheCfg)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeAutoScalingGroups", DescribeAutoScalingGroupsTTL)
+	cacheCfg.SetCacheTTL("autoscaling", "DescribeWarmPool", DescribeWarmPoolTTL)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeLaunchConfigurations", DescribeLaunchConfigurationsTTL)
 	cacheCfg.SetCacheTTL("autoscaling", "DescribeLifecycleHooks", DescribeLifecycleHooksTTL)
 	sess.Handlers.Complete.PushFront(func(r *request.Request) {

controllers/providers/aws/predicates.go

@@ -98,3 +98,10 @@ func IsUsingMixedInstances(group *autoscaling.Group) bool {
 	}
 	return false
 }
+
+func IsUsingWarmPool(group *autoscaling.Group) bool {
+	if group.WarmPoolConfiguration != nil {
+		return true
+	}
+	return false
+}

controllers/provisioners/eks/eks.go

@@ -53,8 +53,9 @@ var (
 	InstanceMgrLifecycleLabel = "instancemgr.keikoproj.io/lifecycle"
 	InstanceMgrImageLabel     = "instancemgr.keikoproj.io/image"
 
-	DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
-	CNIManagedPolicy       = "AmazonEKS_CNI_Policy"
+	DefaultManagedPolicies    = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"}
+	CNIManagedPolicy          = "AmazonEKS_CNI_Policy"
+	AutoscalingReadOnlyPolicy = "AutoScalingReadOnlyAccess"
 )
 
 // New constructs a new instance group provisioner of EKS type

controllers/provisioners/eks/eks_test.go

@@ -94,6 +94,21 @@ func MockEksCluster(version string) *eks.Cluster {
 	}
 }
 
+func MockWarmPoolSpec(maxSize, minSize int64) *v1alpha1.WarmPoolSpec {
+	return &v1alpha1.WarmPoolSpec{
+		MaxSize: maxSize,
+		MinSize: minSize,
+	}
+}
+
+func MockWarmPool(maxSize, minSize int64, status string) *autoscaling.WarmPoolConfiguration {
+	return &autoscaling.WarmPoolConfiguration{
+		MaxGroupPreparedCapacity: aws.Int64(maxSize),
+		MinSize:                  aws.Int64(minSize),
+		Status:                   aws.String(status),
+	}
+}
+
 func MockKubernetesClientSet() kubeprovider.KubernetesClientSet {
 	return kubeprovider.KubernetesClientSet{
 		Kubernetes:  fake.NewSimpleClientset(),
@@ -483,13 +498,20 @@ type MockAutoScalingClient struct {
 	DescribeLifecycleHooksErr              error
 	PutLifecycleHookErr                    error
 	DeleteLifecycleHookErr                 error
+	DescribeWarmPoolErr                    error
+	DeleteWarmPoolErr                      error
+	PutWarmPoolErr                         error
 	DeleteLaunchConfigurationCallCount     int
 	PutLifecycleHookCallCount              int
 	DeleteLifecycleHookCallCount           int
+	PutWarmPoolCallCount                   int
+	DeleteWarmPoolCallCount                int
+	DescribeWarmPoolCallCount              int
 	LaunchConfiguration                    *autoscaling.LaunchConfiguration
 	LaunchConfigurations                   []*autoscaling.LaunchConfiguration
 	AutoScalingGroup                       *autoscaling.Group
 	AutoScalingGroups                      []*autoscaling.Group
+	WarmPoolInstances                      []*autoscaling.Instance
 	LifecycleHooks                         []*autoscaling.LifecycleHook
 }
 
@@ -582,6 +604,21 @@ func (a *MockAutoScalingClient) PutLifecycleHook(input *autoscaling.PutLifecycle
 	return &autoscaling.PutLifecycleHookOutput{}, a.PutLifecycleHookErr
 }
 
+func (a *MockAutoScalingClient) DescribeWarmPool(input *autoscaling.DescribeWarmPoolInput) (*autoscaling.DescribeWarmPoolOutput, error) {
+	a.DescribeWarmPoolCallCount++
+	return &autoscaling.DescribeWarmPoolOutput{Instances: a.WarmPoolInstances}, a.DescribeWarmPoolErr
+}
+
+func (a *MockAutoScalingClient) DeleteWarmPool(input *autoscaling.DeleteWarmPoolInput) (*autoscaling.DeleteWarmPoolOutput, error) {
+	a.DeleteWarmPoolCallCount++
+	return &autoscaling.DeleteWarmPoolOutput{}, a.DeleteWarmPoolErr
+}
+
+func (a *MockAutoScalingClient) PutWarmPool(input *autoscaling.PutWarmPoolInput) (*autoscaling.PutWarmPoolOutput, error) {
+	a.PutWarmPoolCallCount++
+	return &autoscaling.PutWarmPoolOutput{}, a.PutWarmPoolErr
+}
+
 type MockEc2Client struct {
 	ec2iface.EC2API
 	DescribeSubnetsErr                   error

controllers/provisioners/eks/helpers.go

@@ -159,6 +159,15 @@ mount
 echo "{{ .Device}}    {{ .Mount }}    {{ .FileSystem | ToLower }}    defaults    0    2" >> /etc/fstab
 {{- end}}
 {{- end}}
+if [[ $(type -P $(which aws)) ]] && [[ $(type -P $(which jq)) ]] ; then
+	INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)
+	REGION=$(curl http://169.254.169.254/latest/meta-data/placement/region)
+	LIFECYCLE=$(aws autoscaling describe-auto-scaling-instances --region $REGION --instance-id $INSTANCE_ID | jq ".AutoScalingInstances[].LifecycleState" || true)
+	if [[ $LIFECYCLE == *"Warmed"* ]]; then
+		rm /var/lib/cloud/instances/$INSTANCE_ID/sem/config_scripts_user
+		exit 0
+	fi
+fi
 set -o xtrace
 /etc/eks/bootstrap.sh {{ .ClusterName }} {{ .Arguments }}
 set +o xtrace
@@ -839,6 +848,64 @@ func (ctx *EksInstanceGroupContext) GetAddedHooks() ([]v1alpha1.LifecycleHookSpe
 	return addHooks, true
 }
 
+func (ctx *EksInstanceGroupContext) UpdateWarmPool(asgName string) error {
+	var (
+		instanceGroup  = ctx.GetInstanceGroup()
+		spec           = instanceGroup.GetEKSSpec()
+		state          = ctx.GetDiscoveredState()
+		scalingGroup   = state.GetScalingGroup()
+		warmPoolConfig = scalingGroup.WarmPoolConfiguration
+	)
+
+	var warmPoolConfigured bool
+
+	if warmPoolConfig != nil {
+		warmPoolConfigured = true
+	}
+
+	// spec has no warm pool
+	if !spec.HasWarmPool() {
+		// scaling group has warm pool
+		if warmPoolConfigured {
+			// it's already deleting
+			if aws.StringValue(warmPoolConfig.Status) == autoscaling.WarmPoolStatusPendingDelete {
+				return nil
+			}
+			// delete it
+			if err := ctx.AwsWorker.DeleteWarmPool(asgName); err != nil {
+				return errors.Wrapf(err, "failed to delete warm pool for scaling group %v", asgName)
+			}
+		}
+		return nil
+	} else { // warm pool exists in spec
+
+		// check if update/create is needed
+		var (
+			updateRequired bool
+			max            = spec.WarmPool.MaxSize
+			min            = spec.WarmPool.MinSize
+		)
+
+		if warmPoolConfigured {
+			if min != aws.Int64Value(warmPoolConfig.MinSize) {
+				updateRequired = true
+			}
+			if max != aws.Int64Value(warmPoolConfig.MaxGroupPreparedCapacity) {
+				updateRequired = true
+			}
+		}
+
+		// update or create warm pool
+		if updateRequired || !warmPoolConfigured {
+			if err := ctx.AwsWorker.UpdateWarmPool(asgName, min, max); err != nil {
+				return errors.Wrapf(err, "failed to delete warm pool for scaling group %v", asgName)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (ctx *EksInstanceGroupContext) UpdateLifecycleHooks(asgName string) error {
 	var (
 		instanceGroup = ctx.GetInstanceGroup()
@@ -887,6 +954,7 @@ func (ctx *EksInstanceGroupContext) UpdateLifecycleHooks(asgName string) error {
 func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []string) []string {
 	var (
 		instanceGroup = ctx.GetInstanceGroup()
+		spec          = instanceGroup.GetEKSSpec()
 		annotations   = instanceGroup.GetAnnotations()
 	)
 
@@ -917,6 +985,10 @@ func (ctx *EksInstanceGroupContext) GetManagedPoliciesList(additionalPolicies []
 		managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, CNIManagedPolicy))
 	}
 
+	if spec.HasWarmPool() {
+		managedPolicies = append(managedPolicies, fmt.Sprintf("%s/%s", awsprovider.IAMPolicyPrefix, AutoscalingReadOnlyPolicy))
+	}
+
 	return managedPolicies
 }