Add support for specifying maxPods as a bootstrap option (#216)

* Add support for specifying maxPods as a bootstrap option Signed-off-by: Jonah Back <[email protected]> * Fix formatting Signed-off-by: Jonah Back <[email protected]> * Fix crd Signed-off-by: Jonah Back <[email protected]> * Add maxPods support for AL2 Signed-off-by: Jonah Back <[email protected]> Co-authored-by: Eytan Avisror <[email protected]>
keikoproj · Dec 2, 2020 · d796455 · d796455
1 parent 6565d8a
commit d796455
Show file tree

Hide file tree

Showing 6 changed files with 129 additions and 11 deletions.
diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go
@@ -184,6 +184,10 @@ type EKSManagedSpec struct {
 	EKSManagedConfiguration *EKSManagedConfiguration `json:"configuration"`
 }
 
+type BootstrapOptions struct {
+	MaxPods int64 `json:"maxPods,omitempty"`
+}
+
 type EKSSpec struct {
 	MaxSize          int64                    `json:"maxSize,omitempty"`
 	MinSize          int64                    `json:"minSize,omitempty"`
@@ -201,6 +205,7 @@ type EKSConfiguration struct {
 	Subnets                     []string                  `json:"subnets,omitempty"`
 	SuspendedProcesses          []string                  `json:"suspendProcesses,omitempty"`
 	BootstrapArguments          string                    `json:"bootstrapArguments,omitempty"`
+	BootstrapOptions            *BootstrapOptions         `json:"bootstrapOptions,omitempty"`
 	SpotPrice                   string                    `json:"spotPrice,omitempty"`
 	Tags                        []map[string]string       `json:"tags,omitempty"`
 	Labels                      map[string]string         `json:"labels,omitempty"`
@@ -724,6 +729,11 @@ func (c *EKSConfiguration) GetVolumes() []NodeVolume {
 func (c *EKSConfiguration) GetBootstrapArguments() string {
 	return c.BootstrapArguments
 }
+
+func (c *EKSConfiguration) GetBootstrapOptions() *BootstrapOptions {
+	return c.BootstrapOptions
+}
+
 func (c *EKSConfiguration) GetSecurityGroups() []string {
 	if c.NodeSecurityGroups == nil {
 		return []string{}

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml b/config/crd/bases/instancemgr.keikoproj.io_instancegroups.yaml
@@ -77,6 +77,12 @@ spec:
                   properties:
                     bootstrapArguments:
                       type: string
+                    bootstrapOptions:
+                      properties:
+                        maxPods:
+                          format: int64
+                          type: integer
+                      type: object
                     clusterName:
                       type: string
                     image:

diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go
@@ -114,6 +114,7 @@ type EKSUserData struct {
 	PreBootstrap     []string
 	PostBootstrap    []string
 	MountOptions     []MountOpts
+	MaxPods          int64
 }
 
 func (ctx *EksInstanceGroupContext) GetInstanceGroup() *v1alpha1.InstanceGroup {

diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go
@@ -99,16 +99,20 @@ func (ctx *EksInstanceGroupContext) ResolveSecurityGroups() []string {
 
 func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, kubeletExtraArgs string, payload UserDataPayload, mounts []MountOpts) string {
 	var (
-		instanceGroup = ctx.GetInstanceGroup()
-		configuration = instanceGroup.GetEKSConfiguration()
-		state         = ctx.GetDiscoveredState()
-		apiEndpoint   = state.GetClusterEndpoint()
-		clusterCa     = state.GetClusterCA()
-		osFamily      = ctx.GetOsFamily()
-		nodeLabels    = ctx.GetComputedLabels()
-		nodeTaints    = configuration.GetTaints()
+		instanceGroup    = ctx.GetInstanceGroup()
+		configuration    = instanceGroup.GetEKSConfiguration()
+		state            = ctx.GetDiscoveredState()
+		apiEndpoint      = state.GetClusterEndpoint()
+		clusterCa        = state.GetClusterCA()
+		osFamily         = ctx.GetOsFamily()
+		nodeLabels       = ctx.GetComputedLabels()
+		nodeTaints       = configuration.GetTaints()
+		bootstrapOptions = configuration.GetBootstrapOptions()
 	)
-
+	var maxPods int64 = 0
+	if bootstrapOptions != nil {
+		maxPods = bootstrapOptions.MaxPods
+	}
 	var UserDataTemplate string
 	switch strings.ToLower(osFamily) {
 	case OsFamilyWindows:
@@ -125,6 +129,9 @@ func (ctx *EksInstanceGroupContext) GetBasicUserData(clusterName, args string, k
 api-server   = "{{ .ApiEndpoint }}"
 cluster-certificate = "{{ .ClusterCA }}"
 cluster-name = "{{ .ClusterName }}"
+{{- if .MaxPods}}
+max-pods = {{ .MaxPods }}
+{{- end}}
 [settings.kubernetes.node-labels]
 {{- range $key, $value := .NodeLabels }}
 "{{ $key }}" = "{{ $value }}"
@@ -156,6 +163,7 @@ set +o xtrace
 		ApiEndpoint:      apiEndpoint,
 		ClusterCA:        clusterCa,
 		ClusterName:      clusterName,
+		MaxPods:          maxPods,
 		NodeLabels:       nodeLabels,
 		NodeTaints:       nodeTaints,
 		KubeletExtraArgs: kubeletExtraArgs,
@@ -445,7 +453,17 @@ func (ctx *EksInstanceGroupContext) GetLabelList() []string {
 }
 
 func (ctx *EksInstanceGroupContext) GetBootstrapArgs() string {
-	return fmt.Sprintf("--kubelet-extra-args '%v'", ctx.GetKubeletExtraArgs())
+	var (
+		instanceGroup = ctx.GetInstanceGroup()
+		configuration = instanceGroup.GetEKSConfiguration()
+	)
+	var sb strings.Builder
+
+	if configuration.BootstrapOptions != nil && configuration.BootstrapOptions.MaxPods > 0 {
+		sb.WriteString("--use-max-pods false ")
+	}
+	sb.WriteString(fmt.Sprintf("--kubelet-extra-args '%v' ", ctx.GetKubeletExtraArgs()))
+	return sb.String()
 }
 
 func (ctx *EksInstanceGroupContext) GetKubeletExtraArgs() string {
@@ -457,7 +475,12 @@ func (ctx *EksInstanceGroupContext) GetKubeletExtraArgs() string {
 
 	labelsFlag := fmt.Sprintf("--node-labels=%v", strings.Join(ctx.GetLabelList(), ","))
 	taintsFlag := fmt.Sprintf("--register-with-taints=%v", strings.Join(ctx.GetTaintList(), ","))
-	return fmt.Sprintf("%v %v %v", labelsFlag, taintsFlag, bootstrapArgs)
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("%v %v %v ", labelsFlag, taintsFlag, bootstrapArgs))
+	if configuration.BootstrapOptions != nil && configuration.BootstrapOptions.MaxPods > 0 {
+		sb.WriteString(fmt.Sprintf("--max-pods=%v ", configuration.BootstrapOptions.MaxPods))
+	}
+	return sb.String()
 }
 
 func (ctx *EksInstanceGroupContext) discoverSpotPrice() error {

diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go
@@ -506,6 +506,64 @@ func TestGetUserDataStages(t *testing.T) {
 	}
 }
 
+func TestMaxPodsSetCorrectly(t *testing.T) {
+	var (
+		k                         = MockKubernetesClientSet()
+		bottleRocketIgWithMaxPods = MockBottleRocketInstanceGroup()
+		bottleRocketIg            = MockBottleRocketInstanceGroup()
+		amazonLinuxIgWithMaxPods  = MockInstanceGroup()
+		asgMock                   = NewAutoScalingMocker()
+		iamMock                   = NewIamMocker()
+		eksMock                   = NewEksMocker()
+		ec2Mock                   = NewEc2Mocker()
+	)
+
+	w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock)
+
+	bottleRocketIgWithMaxPods.Spec.EKSSpec.EKSConfiguration.BootstrapOptions = &v1alpha1.BootstrapOptions{
+		MaxPods: 15,
+	}
+
+	amazonLinuxIgWithMaxPods.Spec.EKSSpec.EKSConfiguration.BootstrapOptions = &v1alpha1.BootstrapOptions{
+		MaxPods: 15,
+	}
+
+	tests := []struct {
+		ig                         *v1alpha1.InstanceGroup
+		expectedScriptSubstrings   string
+		unexpectedScriptSubstrings string
+	}{
+		{
+			ig:                       bottleRocketIgWithMaxPods,
+			expectedScriptSubstrings: "max-pods = 15",
+		},
+		{
+			ig:                       amazonLinuxIgWithMaxPods,
+			expectedScriptSubstrings: "--max-pods=15",
+		},
+		{
+			ig:                         bottleRocketIg,
+			expectedScriptSubstrings:   "",
+			unexpectedScriptSubstrings: "max-pods",
+		},
+	}
+
+	for i, tc := range tests {
+		t.Logf("Test #%v - %+v", i, tc)
+		ctx := MockContext(tc.ig, k, w)
+		args := ctx.GetBootstrapArgs()
+		basicUserData := ctx.GetBasicUserData("", args, "", UserDataPayload{}, []MountOpts{})
+		basicUserDataDecoded, _ := base64.StdEncoding.DecodeString(basicUserData)
+		basicUserDataString := string(basicUserDataDecoded)
+		if !strings.Contains(basicUserDataString, tc.expectedScriptSubstrings) {
+			t.Fatalf("Cound not find expected string %v script in %v", tc.expectedScriptSubstrings, basicUserDataString)
+		}
+		if tc.unexpectedScriptSubstrings != "" && strings.Contains(basicUserDataString, tc.unexpectedScriptSubstrings) {
+			t.Fatalf("Found unexpected string %v script in %v", tc.unexpectedScriptSubstrings, basicUserDataString)
+		}
+	}
+}
+
 func TestBootstrapDataForOSFamily(t *testing.T) {
 	var (
 		k              = MockKubernetesClientSet()