From 8bda944a3808c0b42ab6a1c652f91a5b882009e7 Mon Sep 17 00:00:00 2001 From: Ole Markus With Date: Thu, 20 Aug 2020 12:50:17 +0200 Subject: [PATCH] Implement cluster autoscaler as bootstrap addon Use provider-agnostic node definition for cas instead of aws auto-discovery Validate clusterAutoscalerSpec Add spec documentation Add cas docs Make CRDs Apply suggestions from code review Co-authored-by: John Gardiner Myers Add enabled flag to cas config Apply suggestions from code review Co-authored-by: Guy Templeton --- docs/cluster_spec.md | 17 ++ go.sum | 1 + k8s/crds/kops.k8s.io_clusters.yaml | 22 ++ pkg/apis/kops/cluster.go | 3 + pkg/apis/kops/componentconfig.go | 23 +++ pkg/apis/kops/v1alpha2/cluster.go | 3 + pkg/apis/kops/v1alpha2/componentconfig.go | 23 +++ .../kops/v1alpha2/zz_generated.conversion.go | 58 ++++++ .../kops/v1alpha2/zz_generated.deepcopy.go | 51 +++++ pkg/apis/kops/validation/validation.go | 18 ++ pkg/apis/kops/zz_generated.deepcopy.go | 51 +++++ pkg/model/components/BUILD.bazel | 1 + pkg/model/components/clusterautoscaler.go | 52 +++++ upup/models/bindata.go | 195 ++++++++++++++++++ .../k8s-1.15.yaml.template | 174 ++++++++++++++++ .../pkg/fi/cloudup/bootstrapchannelbuilder.go | 21 ++ upup/pkg/fi/cloudup/populate_cluster_spec.go | 1 + upup/pkg/fi/cloudup/template_functions.go | 33 +++ 18 files changed, 747 insertions(+) create mode 100644 pkg/model/components/clusterautoscaler.go create mode 100644 upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index b85aa0e7f8d0a..617f90872a995 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -602,6 +602,23 @@ spec: For more details on `horizontalPodAutoscaler` flags see the [official HPA docs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and the [Kops guides on how to set it up](horizontal_pod_autoscaling.md). +## Cluster autoscaler + +{{ kops_feature_table(kops_added_default='1.19', k8s_min='1.15') }} + +Cluster autoscaler can be enabled to automatically adjust the size of the kubernetes cluster. + +```yaml +spec: + cluster: + clusterAutoscaler: + enabled: true + skipNodesWithLocalStorage: true + skipNodesWithSystemPods: true +``` + +Read more about cluster autoscaler in the [official documentation](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler). + ### Feature Gates Feature gates can be configured on the kubelet. diff --git a/go.sum b/go.sum index 3f8101595c10c..33315bed295ae 100644 --- a/go.sum +++ b/go.sum @@ -1278,6 +1278,7 @@ k8s.io/cloud-provider v0.19.0/go.mod h1:TYh7b7kQ6wiqF7Ftb+u3lN4IwvgOPbBrcvC3TDAW k8s.io/cloud-provider-openstack v1.18.0 h1:v/ebjNEdx0hBaygsRohSS643f41lj2CwvapCbn+aLOs= k8s.io/cloud-provider-openstack v1.18.0/go.mod h1:03202t5Sp+4Vmk6pxJ/hVH0fEkm9gMc/pku/QpkJQMQ= k8s.io/cluster-bootstrap v0.19.0/go.mod h1:kBn1DKyqoM245wzz+AAnGkuysJ+9GqVbPYveTo4KiaA= +k8s.io/code-generator v0.19.0 h1:r0BxYnttP/r8uyKd4+Njg0B57kKi8wLvwEzaaVy3iZ8= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/component-base v0.19.0 h1:OueXf1q3RW7NlLlUCj2Dimwt7E1ys6ZqRnq53l2YuoE= k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index edb0725507878..8c0211036f025 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -335,6 +335,28 @@ spec: cloudProvider: description: The CloudProvider to use (aws or gce) type: string + clusterAutoscaler: + description: ClusterAutoscaler defines the cluaster autoscaler configuration. + properties: + balanceSimilarNodeGroups: + description: 'BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one. Default: false' + type: boolean + enabled: + description: 'Enabled enables the cluster autoscaler. Default: false' + type: boolean + expander: + description: 'Expander determines the strategy for which instance group gets expanded. Supported values: least-waste, most-pods, random. Default: least-waste' + type: string + scaleDownUtilizationThreshold: + description: 'ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. Default: 0.5' + type: string + skipNodesWithLocalStorage: + description: 'SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage. Default: true' + type: boolean + skipNodesWithSystemPods: + description: 'SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. Default: true' + type: boolean + type: object clusterDNSDomain: description: ClusterDNSDomain is the suffix we use for internal DNS names (normally cluster.local) type: string diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go index 60f0aac349f59..bf0aecca71e1b 100644 --- a/pkg/apis/kops/cluster.go +++ b/pkg/apis/kops/cluster.go @@ -191,6 +191,9 @@ type ClusterSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the default rolling-update settings for instance groups RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + + // ClusterAutoscaler defines the cluster autoscaler configuration. + ClusterAutoscaler *ClusterAutoscalerConfig `json:"clusterAutoscaler,omitempty"` } // NodeAuthorizationSpec is used to node authorization diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go index 66f84bba33ed9..4e4792ee45cc1 100644 --- a/pkg/apis/kops/componentconfig.go +++ b/pkg/apis/kops/componentconfig.go @@ -759,6 +759,29 @@ type CloudConfiguration struct { Openstack *OpenstackConfiguration `json:"openstack,omitempty"` } +// ClusterAutoscalerConfig determines the cluster autoscaler configuration. +type ClusterAutoscalerConfig struct { + // Enabled enables the cluster autoscaler. + // Default: false + Enabled *bool `json:"enabled,omitempty"` + // Expander determines the strategy for which instance group gets expanded. + // Supported values: least-waste, most-pods, random. + // Default: least-waste + Expander *string `json:"expander,omitempty"` + // BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one. + // Default: false + BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"` + // ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. + // Default: 0.5 + ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"` + // SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. + // Default: true + SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"` + // SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage. + // Default: true + SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"` +} + // HasAdmissionController checks if a specific admission controller is enabled func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool { for _, x := range c.AdmissionControl { diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go index 81e59b3f3e7c8..e296033bee42d 100644 --- a/pkg/apis/kops/v1alpha2/cluster.go +++ b/pkg/apis/kops/v1alpha2/cluster.go @@ -189,6 +189,9 @@ type ClusterSpec struct { SysctlParameters []string `json:"sysctlParameters,omitempty"` // RollingUpdate defines the default rolling-update settings for instance groups RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` + + // ClusterAutoscaler defines the cluaster autoscaler configuration. + ClusterAutoscaler *ClusterAutoscalerConfig `json:"clusterAutoscaler,omitempty"` } // NodeAuthorizationSpec is used to node authorization diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go index bfa365c288cbe..b30421ff1473b 100644 --- a/pkg/apis/kops/v1alpha2/componentconfig.go +++ b/pkg/apis/kops/v1alpha2/componentconfig.go @@ -760,6 +760,29 @@ type CloudConfiguration struct { Openstack *OpenstackConfiguration `json:"openstack,omitempty"` } +// ClusterAutoscalerConfig determines the cluster autoscaler configuration. +type ClusterAutoscalerConfig struct { + // Enabled enables the cluster autoscaler. + // Default: false + Enabled *bool `json:"enabled,omitempty"` + // Expander determines the strategy for which instance group gets expanded. + // Supported values: least-waste, most-pods, random. + // Default: least-waste + Expander *string `json:"expander,omitempty"` + // BalanceSimilarNodeGroups makes cluster autoscaler treat similar node groups as one. + // Default: false + BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"` + // ScaleDownUtilizationThreshold determines the utilization threshold for node scale-down. + // Default: 0.5 + ScaleDownUtilizationThreshold *string `json:"scaleDownUtilizationThreshold,omitempty"` + // SkipNodesWithSystemPods makes cluster autoscaler skip scale-down of nodes with non-DaemonSet pods in the kube-system namespace. + // Default: true + SkipNodesWithSystemPods *bool `json:"skipNodesWithSystemPods,omitempty"` + // SkipNodesWithLocalStorage makes cluster autoscaler skip scale-down of nodes with local storage. + // Default: true + SkipNodesWithLocalStorage *bool `json:"skipNodesWithLocalStorage,omitempty"` +} + // HasAdmissionController checks if a specific admission controller is enabled func (c *KubeAPIServerConfig) HasAdmissionController(name string) bool { for _, x := range c.AdmissionControl { diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index df09dce66f906..d9cebe8013eaa 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -213,6 +213,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*ClusterAutoscalerConfig)(nil), (*kops.ClusterAutoscalerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig(a.(*ClusterAutoscalerConfig), b.(*kops.ClusterAutoscalerConfig), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*kops.ClusterAutoscalerConfig)(nil), (*ClusterAutoscalerConfig)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig(a.(*kops.ClusterAutoscalerConfig), b.(*ClusterAutoscalerConfig), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*ClusterList)(nil), (*kops.ClusterList)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_ClusterList_To_kops_ClusterList(a.(*ClusterList), b.(*kops.ClusterList), scope) }); err != nil { @@ -1730,6 +1740,36 @@ func Convert_kops_Cluster_To_v1alpha2_Cluster(in *kops.Cluster, out *Cluster, s return autoConvert_kops_Cluster_To_v1alpha2_Cluster(in, out, s) } +func autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig(in *ClusterAutoscalerConfig, out *kops.ClusterAutoscalerConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Expander = in.Expander + out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups + out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods + out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage + return nil +} + +// Convert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig is an autogenerated conversion function. +func Convert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig(in *ClusterAutoscalerConfig, out *kops.ClusterAutoscalerConfig, s conversion.Scope) error { + return autoConvert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig(in, out, s) +} + +func autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig(in *kops.ClusterAutoscalerConfig, out *ClusterAutoscalerConfig, s conversion.Scope) error { + out.Enabled = in.Enabled + out.Expander = in.Expander + out.BalanceSimilarNodeGroups = in.BalanceSimilarNodeGroups + out.ScaleDownUtilizationThreshold = in.ScaleDownUtilizationThreshold + out.SkipNodesWithSystemPods = in.SkipNodesWithSystemPods + out.SkipNodesWithLocalStorage = in.SkipNodesWithLocalStorage + return nil +} + +// Convert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig is an autogenerated conversion function. +func Convert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig(in *kops.ClusterAutoscalerConfig, out *ClusterAutoscalerConfig, s conversion.Scope) error { + return autoConvert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig(in, out, s) +} + func autoConvert_v1alpha2_ClusterList_To_kops_ClusterList(in *ClusterList, out *kops.ClusterList, s conversion.Scope) error { out.ListMeta = in.ListMeta if in.Items != nil { @@ -2086,6 +2126,15 @@ func autoConvert_v1alpha2_ClusterSpec_To_kops_ClusterSpec(in *ClusterSpec, out * } else { out.RollingUpdate = nil } + if in.ClusterAutoscaler != nil { + in, out := &in.ClusterAutoscaler, &out.ClusterAutoscaler + *out = new(kops.ClusterAutoscalerConfig) + if err := Convert_v1alpha2_ClusterAutoscalerConfig_To_kops_ClusterAutoscalerConfig(*in, *out, s); err != nil { + return err + } + } else { + out.ClusterAutoscaler = nil + } return nil } @@ -2408,6 +2457,15 @@ func autoConvert_kops_ClusterSpec_To_v1alpha2_ClusterSpec(in *kops.ClusterSpec, } else { out.RollingUpdate = nil } + if in.ClusterAutoscaler != nil { + in, out := &in.ClusterAutoscaler, &out.ClusterAutoscaler + *out = new(ClusterAutoscalerConfig) + if err := Convert_kops_ClusterAutoscalerConfig_To_v1alpha2_ClusterAutoscalerConfig(*in, *out, s); err != nil { + return err + } + } else { + out.ClusterAutoscaler = nil + } return nil } diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go index e517534519451..b9092d42686cc 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go @@ -575,6 +575,52 @@ func (in *Cluster) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Expander != nil { + in, out := &in.Expander, &out.Expander + *out = new(string) + **out = **in + } + if in.BalanceSimilarNodeGroups != nil { + in, out := &in.BalanceSimilarNodeGroups, &out.BalanceSimilarNodeGroups + *out = new(bool) + **out = **in + } + if in.ScaleDownUtilizationThreshold != nil { + in, out := &in.ScaleDownUtilizationThreshold, &out.ScaleDownUtilizationThreshold + *out = new(string) + **out = **in + } + if in.SkipNodesWithSystemPods != nil { + in, out := &in.SkipNodesWithSystemPods, &out.SkipNodesWithSystemPods + *out = new(bool) + **out = **in + } + if in.SkipNodesWithLocalStorage != nil { + in, out := &in.SkipNodesWithLocalStorage, &out.SkipNodesWithLocalStorage + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAutoscalerConfig. +func (in *ClusterAutoscalerConfig) DeepCopy() *ClusterAutoscalerConfig { + if in == nil { + return nil + } + out := new(ClusterAutoscalerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterList) DeepCopyInto(out *ClusterList) { *out = *in @@ -859,6 +905,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.ClusterAutoscaler != nil { + in, out := &in.ClusterAutoscaler, &out.ClusterAutoscaler + *out = new(ClusterAutoscalerConfig) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index f62d0af2425a3..e96aa947e0abe 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -150,6 +150,10 @@ func validateClusterSpec(spec *kops.ClusterSpec, c *kops.Cluster, fieldPath *fie allErrs = append(allErrs, validateNodeAuthorization(spec.NodeAuthorization, c, fieldPath.Child("nodeAuthorization"))...) } + if spec.ClusterAutoscaler != nil { + allErrs = append(allErrs, validateClusterAutoscaler(c, spec.ClusterAutoscaler, fieldPath.Child("clusterAutoscaler"))...) + } + // IAM additionalPolicies if spec.AdditionalPolicies != nil { for k, v := range *spec.AdditionalPolicies { @@ -1113,3 +1117,17 @@ func validateNodeLocalDNS(spec *kops.ClusterSpec, fldpath *field.Path) field.Err return allErrs } + +func validateClusterAutoscaler(cluster *kops.Cluster, spec *kops.ClusterAutoscalerConfig, fldPath *field.Path) (allErrs field.ErrorList) { + if !cluster.IsKubernetesGTE("1.15") || cluster.IsKubernetesGTE("1.20") { + allErrs = append(allErrs, field.Forbidden(fldPath, "Cluster autoscaler requires kubernetesVersion between 1.11 and 1.19")) + } + + allErrs = append(allErrs, IsValidValue(fldPath.Child("expander"), spec.Expander, []string{"least-waste", "random", "most-pods"})...) + + if kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderOpenstack { + allErrs = append(allErrs, field.Forbidden(fldPath, "Cluster autoscaler is not supported on OpenStack")) + } + + return allErrs +} diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go index 16088845c2c5c..8c395e877b35e 100644 --- a/pkg/apis/kops/zz_generated.deepcopy.go +++ b/pkg/apis/kops/zz_generated.deepcopy.go @@ -675,6 +675,52 @@ func (in *Cluster) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterAutoscalerConfig) DeepCopyInto(out *ClusterAutoscalerConfig) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + if in.Expander != nil { + in, out := &in.Expander, &out.Expander + *out = new(string) + **out = **in + } + if in.BalanceSimilarNodeGroups != nil { + in, out := &in.BalanceSimilarNodeGroups, &out.BalanceSimilarNodeGroups + *out = new(bool) + **out = **in + } + if in.ScaleDownUtilizationThreshold != nil { + in, out := &in.ScaleDownUtilizationThreshold, &out.ScaleDownUtilizationThreshold + *out = new(string) + **out = **in + } + if in.SkipNodesWithSystemPods != nil { + in, out := &in.SkipNodesWithSystemPods, &out.SkipNodesWithSystemPods + *out = new(bool) + **out = **in + } + if in.SkipNodesWithLocalStorage != nil { + in, out := &in.SkipNodesWithLocalStorage, &out.SkipNodesWithLocalStorage + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAutoscalerConfig. +func (in *ClusterAutoscalerConfig) DeepCopy() *ClusterAutoscalerConfig { + if in == nil { + return nil + } + out := new(ClusterAutoscalerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterList) DeepCopyInto(out *ClusterList) { *out = *in @@ -959,6 +1005,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(RollingUpdate) (*in).DeepCopyInto(*out) } + if in.ClusterAutoscaler != nil { + in, out := &in.ClusterAutoscaler, &out.ClusterAutoscaler + *out = new(ClusterAutoscalerConfig) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/model/components/BUILD.bazel b/pkg/model/components/BUILD.bazel index b27123957d0a1..f489f3768498b 100644 --- a/pkg/model/components/BUILD.bazel +++ b/pkg/model/components/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "apiserver.go", "cilium.go", + "clusterautoscaler.go", "containerd.go", "context.go", "defaults.go", diff --git a/pkg/model/components/clusterautoscaler.go b/pkg/model/components/clusterautoscaler.go new file mode 100644 index 0000000000000..12cebbcdcbae9 --- /dev/null +++ b/pkg/model/components/clusterautoscaler.go @@ -0,0 +1,52 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package components + +import ( + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/loader" +) + +// ClusterAutoscalerOptionsBuilder adds options for cluster autoscaler to the model +type ClusterAutoscalerOptionsBuilder struct { + *OptionsContext +} + +var _ loader.OptionsBuilder = &ClusterAutoscalerOptionsBuilder{} + +func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error { + clusterSpec := o.(*kops.ClusterSpec) + cas := clusterSpec.ClusterAutoscaler + if cas == nil || !fi.BoolValue(cas.Enabled) { + return nil + } + + if cas.Expander == nil { + cas.Expander = fi.String("random") + } + if cas.ScaleDownUtilizationThreshold == nil { + cas.ScaleDownUtilizationThreshold = fi.String("0.5") + } + if cas.SkipNodesWithLocalStorage == nil { + cas.SkipNodesWithLocalStorage = fi.Bool(true) + } + if cas.SkipNodesWithSystemPods == nil { + cas.SkipNodesWithSystemPods = fi.Bool(true) + } + return nil +} diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 9957aa38134c2..0f8dfe167ee4b 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -6,6 +6,7 @@ // upup/models/cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml // upup/models/cloudup/resources/addons/authentication.kope.io/k8s-1.8.yaml +// upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template // upup/models/cloudup/resources/addons/core.addons.k8s.io/addon.yaml // upup/models/cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template // upup/models/cloudup/resources/addons/core.addons.k8s.io/k8s-1.7.yaml.template @@ -855,6 +856,196 @@ func cloudupResourcesAddonsAuthenticationKopeIoK8s18Yaml() (*asset, error) { return a, nil } +var _cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate = []byte(`# Sourced from https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws/examples +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - operator: "Exists" + key: node-role.kubernetes.io/master + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:{{ ClusterAutoscalerTag }} + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=2 + - --stderrthreshold=info + - --cloud-provider={{ .CloudProvider }} + - --skip-nodes-with-local-storage={{ .ClusterAutoscaler.SkipNodesWithLocalStorage }} + - --expander={{ .ClusterAutoscaler.Expander }} + {{ range $name, $spec := GetNodeInstanceGroups }} + - --nodes={{ $spec.MinSize }}:{{ $spec.MaxSize }}:{{ $name }} + {{ end }} + ports: + - containerPort: 8085 + protocol: TCP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health-check + port: 8085 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1`) + +func cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplateBytes() ([]byte, error) { + return _cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, nil +} + +func cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate() (*asset, error) { + bytes, err := cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplateBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _cloudupResourcesAddonsCoreAddonsK8sIoAddonYaml = []byte(`kind: Addons metadata: name: core @@ -20408,6 +20599,7 @@ var _bindata = map[string]func() (*asset, error){ "cloudup/resources/addons/authentication.aws/k8s-1.12.yaml.template": cloudupResourcesAddonsAuthenticationAwsK8s112YamlTemplate, "cloudup/resources/addons/authentication.kope.io/k8s-1.12.yaml": cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, "cloudup/resources/addons/authentication.kope.io/k8s-1.8.yaml": cloudupResourcesAddonsAuthenticationKopeIoK8s18Yaml, + "cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template": cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, "cloudup/resources/addons/core.addons.k8s.io/addon.yaml": cloudupResourcesAddonsCoreAddonsK8sIoAddonYaml, "cloudup/resources/addons/core.addons.k8s.io/k8s-1.12.yaml.template": cloudupResourcesAddonsCoreAddonsK8sIoK8s112YamlTemplate, "cloudup/resources/addons/core.addons.k8s.io/k8s-1.7.yaml.template": cloudupResourcesAddonsCoreAddonsK8sIoK8s17YamlTemplate, @@ -20524,6 +20716,9 @@ var _bintree = &bintree{nil, map[string]*bintree{ "k8s-1.12.yaml": {cloudupResourcesAddonsAuthenticationKopeIoK8s112Yaml, map[string]*bintree{}}, "k8s-1.8.yaml": {cloudupResourcesAddonsAuthenticationKopeIoK8s18Yaml, map[string]*bintree{}}, }}, + "cluster-autoscaler.addons.k8s.io": {nil, map[string]*bintree{ + "k8s-1.15.yaml.template": {cloudupResourcesAddonsClusterAutoscalerAddonsK8sIoK8s115YamlTemplate, map[string]*bintree{}}, + }}, "core.addons.k8s.io": {nil, map[string]*bintree{ "addon.yaml": {cloudupResourcesAddonsCoreAddonsK8sIoAddonYaml, map[string]*bintree{}}, "k8s-1.12.yaml.template": {cloudupResourcesAddonsCoreAddonsK8sIoK8s112YamlTemplate, map[string]*bintree{}}, diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template new file mode 100644 index 0000000000000..ec9ae588a7870 --- /dev/null +++ b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template @@ -0,0 +1,174 @@ +# Sourced from https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/aws/examples +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - operator: "Exists" + key: node-role.kubernetes.io/master + nodeSelector: + node-role.kubernetes.io/master: "" + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:{{ ClusterAutoscalerTag }} + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --v=2 + - --stderrthreshold=info + - --cloud-provider={{ .CloudProvider }} + - --skip-nodes-with-local-storage={{ .ClusterAutoscaler.SkipNodesWithLocalStorage }} + - --expander={{ .ClusterAutoscaler.Expander }} + {{ range $name, $spec := GetNodeInstanceGroups }} + - --nodes={{ $spec.MinSize }}:{{ $spec.MaxSize }}:{{ $name }} + {{ end }} + ports: + - containerPort: 8085 + protocol: TCP + livenessProbe: + failureThreshold: 3 + httpGet: + path: /health-check + port: 8085 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 \ No newline at end of file diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index f19843d6422df..f263e687fddd3 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -521,6 +521,27 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { } } + if b.Cluster.Spec.ClusterAutoscaler != nil && fi.BoolValue(b.Cluster.Spec.ClusterAutoscaler.Enabled) { + { + key := "cluster-autoscaler.addons.k8s.io" + version := "1.19.0" + + { + location := key + "/k8s-1.15.yaml" + id := "k8s-1.15" + + addons.Spec.Addons = append(addons.Spec.Addons, &channelsapi.AddonSpec{ + Name: fi.String(key), + Version: fi.String(version), + Selector: map[string]string{"k8s-addon": key}, + Manifest: fi.String(location), + KubernetesVersion: ">=1.15.0", + Id: id, + }) + } + } + } + if kops.CloudProviderID(b.Cluster.Spec.CloudProvider) == kops.CloudProviderAWS { key := "storage-aws.addons.k8s.io" version := "1.15.0" diff --git a/upup/pkg/fi/cloudup/populate_cluster_spec.go b/upup/pkg/fi/cloudup/populate_cluster_spec.go index 1ce3e7ea516c3..032b76502f6df 100644 --- a/upup/pkg/fi/cloudup/populate_cluster_spec.go +++ b/upup/pkg/fi/cloudup/populate_cluster_spec.go @@ -278,6 +278,7 @@ func (c *populateClusterSpec) run(clientset simple.Clientset) error { codeModels = append(codeModels, &components.CiliumOptionsBuilder{Context: optionsContext}) codeModels = append(codeModels, &components.OpenStackOptionsBulder{Context: optionsContext}) codeModels = append(codeModels, &components.DiscoveryOptionsBuilder{OptionsContext: optionsContext}) + codeModels = append(codeModels, &components.ClusterAutoscalerOptionsBuilder{OptionsContext: optionsContext}) } } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 9c4c358cf1cd5..4039c944f3028 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -92,6 +92,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS } dest["GetInstanceGroup"] = tf.GetInstanceGroup + dest["GetNodeInstanceGroups"] = tf.GetNodeInstanceGroups dest["CloudTags"] = tf.CloudTagsForInstanceGroup dest["KubeDNS"] = func() *kops.KubeDNSConfig { return cluster.Spec.KubeDNS @@ -126,6 +127,7 @@ func (tf *TemplateFunctions) AddTo(dest template.FuncMap, secretStore fi.SecretS // will return openstack external ccm image location for current kubernetes version dest["OpenStackCCMTag"] = tf.OpenStackCCMTag + dest["ClusterAutoscalerTag"] = tf.ClusterAutoscalerTag dest["ProxyEnv"] = tf.ProxyEnv dest["KopsSystemEnv"] = tf.KopsSystemEnv @@ -526,3 +528,34 @@ func (tf *TemplateFunctions) OpenStackCCMTag() string { } return tag } + +// GetNodeInstanceGroups returns a map containing the defined instance groups of role "Node". +func (tf *TemplateFunctions) GetNodeInstanceGroups() map[string]kops.InstanceGroupSpec { + nodegroups := make(map[string]kops.InstanceGroupSpec) + for _, ig := range tf.KopsModelContext.InstanceGroups { + if ig.Spec.Role == kops.InstanceGroupRoleNode { + nodegroups[ig.ObjectMeta.Name] = ig.Spec + } + } + return nodegroups +} + +func (tf *TemplateFunctions) ClusterAutoscalerTag() (tag string) { + tag = "latest" + parsed, err := util.ParseKubernetesVersion(tf.Cluster.Spec.KubernetesVersion) + if err == nil { + switch parsed.Minor { + case 19: + tag = "v1.19.0" + case 18: + tag = "v1.18.2" + case 17: + tag = "v1.17.3" + case 16: + tag = "v1.16.6" + case 15: + tag = "v1.15.7" + } + } + return tag +}