Implement cluster autoscaler as bootstrap addon

Use provider-agnostic node definition for cas instead of aws auto-discovery Validate clusterAutoscalerSpec Add spec documentation Add cas docs Make CRDs Apply suggestions from code review Co-authored-by: John Gardiner Myers <[email protected]> Add enabled flag to cas config Apply suggestions from code review Co-authored-by: Guy Templeton <[email protected]>
kubernetes · Sep 2, 2020 · 8bda944 · 8bda944
1 parent 1579e3a
commit 8bda944
Show file tree

Hide file tree

Showing 18 changed files with 747 additions and 0 deletions.
diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
@@ -602,6 +602,23 @@ spec:
 
 For more details on `horizontalPodAutoscaler` flags see the [official HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and the [Kops guides on how to set it up](horizontal_pod_autoscaling.md).
 
+## Cluster autoscaler
+
+{{ kops_feature_table(kops_added_default='1.19', k8s_min='1.15') }}
+
+Cluster autoscaler can be enabled to automatically adjust the size of the kubernetes cluster.
+
+```yaml
+spec:
+  cluster:
+    clusterAutoscaler:
+      enabled: true
+      skipNodesWithLocalStorage: true
+      skipNodesWithSystemPods: true
+```
+
+Read more about cluster autoscaler in the [official documentation](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler).
+
 ###  Feature Gates
 
 Feature gates can be configured on the kubelet.

diff --git a/go.sum b/go.sum
@@ -1278,6 +1278,7 @@ k8s.io/cloud-provider v0.19.0/go.mod h1:TYh7b7kQ6wiqF7Ftb+u3lN4IwvgOPbBrcvC3TDAW
 k8s.io/cloud-provider-openstack v1.18.0 h1:v/ebjNEdx0hBaygsRohSS643f41lj2CwvapCbn+aLOs=
 k8s.io/cloud-provider-openstack v1.18.0/go.mod h1:03202t5Sp+4Vmk6pxJ/hVH0fEkm9gMc/pku/QpkJQMQ=
 k8s.io/cluster-bootstrap v0.19.0/go.mod h1:kBn1DKyqoM245wzz+AAnGkuysJ+9GqVbPYveTo4KiaA=
+k8s.io/code-generator v0.19.0 h1:r0BxYnttP/r8uyKd4+Njg0B57kKi8wLvwEzaaVy3iZ8=
 k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
 k8s.io/component-base v0.19.0 h1:OueXf1q3RW7NlLlUCj2Dimwt7E1ys6ZqRnq53l2YuoE=
 k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y=

diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -335,6 +335,28 @@ spec:
               cloudProvider:
                 description: The CloudProvider to use (aws or gce)
                 type: string
+              clusterAutoscaler:
+                description: ClusterAutoscaler defines the cluaster autoscaler configuration.
+                properties:
+                  balanceSimilarNodeGroups:
+                    description: 'BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one. Default: false'
+                    type: boolean
+                  enabled:
+                    description: 'Enabled enables the cluster autoscaler. Default: false'
+                    type: boolean
+                  expander:
+                    description: 'Expander determines the strategy for which instance group gets expanded. Supported values: least-waste, most-pods, random. Default: least-waste'
+                    type: string
+                  scaleDownUtilizationThreshold:
+                    description: 'ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. Default: 0.5'
+                    type: string
+                  skipNodesWithLocalStorage:
+                    description: 'SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage. Default: true'
+                    type: boolean
+                  skipNodesWithSystemPods:
+                    description: 'SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. Default: true'
+                    type: boolean
+                type: object
               clusterDNSDomain:
                 description: ClusterDNSDomain is the suffix we use for internal DNS names (normally cluster.local)
                 type: string

diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go
@@ -191,6 +191,9 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+
+	// ClusterAutoscaler defines the cluster autoscaler configuration.
+	ClusterAutoscaler *ClusterAutoscalerConfig `json:"clusterAutoscaler,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go
@@ -759,6 +759,29 @@ type CloudConfiguration struct {
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 }
 
+// ClusterAutoscalerConfig determines the cluster autoscaler configuration.
+type ClusterAutoscalerConfig struct {
+	// Enabled enables the cluster autoscaler.
+	// Default: false
+	Enabled *bool `json:"enabled,omitempty"`
+	// Expander determines the strategy for which instance group gets expanded.
+	// Supported values: least-waste, most-pods, random.
+	// Default: least-waste
+	Expander *string `json:"expander,omitempty"`
+	// BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one.
+	// Default: false
+	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down.
+	// Default: 0.5
+	ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"`
+	// SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
+	// Default: true
+	SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"`
+	// SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage.
+	// Default: true
+	SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"`
+}
+
 // HasAdmissionController checks if a specific admission controller is enabled
 func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool {
 	for _, x := range c.AdmissionControl {

diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go
@@ -189,6 +189,9 @@ type ClusterSpec struct {
 	SysctlParameters []string `json:"sysctlParameters,omitempty"`
 	// RollingUpdate defines the default rolling-update settings for instance groups
 	RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
+
+	// ClusterAutoscaler defines the cluaster autoscaler configuration.
+	ClusterAutoscaler *ClusterAutoscalerConfig `json:"clusterAutoscaler,omitempty"`
 }
 
 // NodeAuthorizationSpec is used to node authorization

diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go
@@ -760,6 +760,29 @@ type CloudConfiguration struct {
 	Openstack *OpenstackConfiguration `json:"openstack,omitempty"`
 }
 
+// ClusterAutoscalerConfig determines the cluster autoscaler configuration.
+type ClusterAutoscalerConfig struct {
+	// Enabled enables the cluster autoscaler.
+	// Default: false
+	Enabled *bool `json:"enabled,omitempty"`
+	// Expander determines the strategy for which instance group gets expanded.
+	// Supported values: least-waste, most-pods, random.
+	// Default: least-waste
+	Expander *string `json:"expander,omitempty"`
+	// BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one.
+	// Default: false
+	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down.
+	// Default: 0.5
+	ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"`
+	// SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace.
+	// Default: true
+	SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"`
+	// SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage.
+	// Default: true
+	SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"`
+}
+
 // HasAdmissionController checks if a specific admission controller is enabled
 func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool {
 	for _, x := range c.AdmissionControl {

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go
@@ -150,6 +150,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie
 		allErrs = append(allErrs, validateNodeAuthorization(spec.NodeAuthorization, c, fieldPath.Child("nodeAuthorization"))...)
 	}
 
+	if spec.ClusterAutoscaler != nil {
+		allErrs = append(allErrs, validateClusterAutoscaler(c, spec.ClusterAutoscaler, fieldPath.Child("clusterAutoscaler"))...)
+	}
+
 	// IAM additionalPolicies
 	if spec.AdditionalPolicies != nil {
 		for k, v := range *spec.AdditionalPolicies {
@@ -1113,3 +1117,17 @@ func validateNodeLocalDNS(spec *kops.ClusterSpec, fldpath *field.Path) field.Err
 
 	return allErrs
 }
+
+func validateClusterAutoscaler(cluster *kops.Cluster, spec *kops.ClusterAutoscalerConfig, fldPath *field.Path) (allErrs field.ErrorList) {
+	if !cluster.IsKubernetesGTE("1.15") || cluster.IsKubernetesGTE("1.20") {
+		allErrs = append(allErrs, field.Forbidden(fldPath, "Cluster autoscaler requires kubernetesVersion between 1.11 and 1.19"))
+	}
+
+	allErrs = append(allErrs, IsValidValue(fldPath.Child("expander"), spec.Expander, []string{"least-waste", "random", "most-pods"})...)
+
+	if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderOpenstack {
+		allErrs = append(allErrs, field.Forbidden(fldPath, "Cluster autoscaler is not supported on OpenStack"))
+	}
+
+	return allErrs
+}
diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
diff --git a/pkg/model/components/BUILD.bazel b/pkg/model/components/BUILD.bazel
@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "apiserver.go",
         "cilium.go",
+        "clusterautoscaler.go",
         "containerd.go",
         "context.go",
         "defaults.go",