From be7ea2fa4370247baba8dabd6088e5e68c53754a Mon Sep 17 00:00:00 2001 From: jan-est Date: Thu, 11 Mar 2021 15:16:36 +0200 Subject: [PATCH] Backport KCP Rollout Strategy to release-0.3 --- .../v1alpha3/kubeadm_control_plane_types.go | 41 ++++++++++++++ .../v1alpha3/kubeadm_control_plane_webhook.go | 56 +++++++++++++++++++ .../kubeadm_control_plane_webhook_test.go | 20 +++++++ .../api/v1alpha3/zz_generated.deepcopy.go | 46 +++++++++++++++ ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 24 ++++++++ .../kubeadm/controllers/controller_test.go | 9 +++ controlplane/kubeadm/controllers/upgrade.go | 21 ++++--- 7 files changed, 208 insertions(+), 9 deletions(-) diff --git a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go index 48f374d20533..070b77c7a662 100644 --- a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go +++ b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go @@ -19,12 +19,21 @@ package v1alpha3 import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" "sigs.k8s.io/cluster-api/errors" ) +type RolloutStrategyType string + +const ( + // Replace the old control planes by new one using rolling update + // i.e. gradually scale up or down the old control planes and scale up or down the new one. + RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate" +) + const ( KubeadmControlPlaneFinalizer = "kubeadm.controlplane.cluster.x-k8s.io" @@ -72,6 +81,38 @@ type KubeadmControlPlaneSpec struct { // NOTE: NodeDrainTimeout is different from `kubectl drain --timeout` // +optional NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"` + + // The RolloutStrategy to use to replace control plane machines with + // new ones. + // +optional + RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"` +} + +// RolloutStrategy describes how to replace existing machines +// with new ones. +type RolloutStrategy struct { + // Type of rollout. Currently the only supported strategy is + // "RollingUpdate". + // Default is RollingUpdate. + // +optional + Type RolloutStrategyType `json:"type,omitempty"` + + // Rolling update config params. Present only if + // RolloutStrategyType = RollingUpdate. + // +optional + RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"` +} + +// RollingUpdate is used to control the desired behavior of rolling update. +type RollingUpdate struct { + // The maximum number of control planes that can be scheduled above or under the + // desired number of control planes. + // Value can be an absolute number 1 or 0. + // Defaults to 1. + // Example: when this is set to 1, the control plane can be scaled + // up immediately when the rolling update starts. + // +optional + MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"` } // KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane. diff --git a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go index 2496629310b5..bc2155f9a78b 100644 --- a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go +++ b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/validation/field" kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" "sigs.k8s.io/cluster-api/util" @@ -64,6 +65,25 @@ func (in *KubeadmControlPlane) Default() { if !strings.HasPrefix(in.Spec.Version, "v") { in.Spec.Version = "v" + in.Spec.Version } + + ios1 := intstr.FromInt(1) + + if in.Spec.RolloutStrategy == nil { + in.Spec.RolloutStrategy = &RolloutStrategy{} + } + + // Enforce RollingUpdate strategy and default MaxSurge if not set. + if in.Spec.RolloutStrategy != nil { + if len(in.Spec.RolloutStrategy.Type) == 0 { + in.Spec.RolloutStrategy.Type = RollingUpdateStrategyType + } + if in.Spec.RolloutStrategy.Type == RollingUpdateStrategyType { + if in.Spec.RolloutStrategy.RollingUpdate == nil { + in.Spec.RolloutStrategy.RollingUpdate = &RollingUpdate{} + } + in.Spec.RolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(in.Spec.RolloutStrategy.RollingUpdate.MaxSurge, ios1) + } + } } // ValidateCreate implements webhook.Validator so a webhook will be registered for the type @@ -119,6 +139,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error { {spec, "version"}, {spec, "upgradeAfter"}, {spec, "nodeDrainTimeout"}, + {spec, "rolloutStrategy"}, } allErrs := in.validateCommon() @@ -272,6 +293,41 @@ func (in *KubeadmControlPlane) validateCommon() (allErrs field.ErrorList) { allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), in.Spec.Version, "must be a valid semantic version")) } + if in.Spec.RolloutStrategy != nil { + if in.Spec.RolloutStrategy.Type != RollingUpdateStrategyType { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "type"), + "only RollingUpdateStrategyType is supported", + ), + ) + } + + ios1 := intstr.FromInt(1) + ios0 := intstr.FromInt(0) + + if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge == ios0 && *in.Spec.Replicas < int32(3) { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "rollingUpdate"), + "when KubeadmControlPlane is configured to scale-in, replica count needs to be at least 3", + ), + ) + } + + if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios1 && *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios0 { + allErrs = append( + allErrs, + field.Required( + field.NewPath("spec", "rolloutStrategy", "rollingUpdate", "maxSurge"), + "value must be 1 or 0", + ), + ) + } + } + allErrs = append(allErrs, in.validateCoreDNSImage()...) return allErrs diff --git a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook_test.go b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook_test.go index 4a1e4dc50114..39604180ec4b 100644 --- a/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook_test.go +++ b/controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook_test.go @@ -24,6 +24,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3" kubeadmv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" @@ -39,12 +40,15 @@ func TestKubeadmControlPlaneDefault(t *testing.T) { Spec: KubeadmControlPlaneSpec{ InfrastructureTemplate: corev1.ObjectReference{}, Version: "1.18.3", + RolloutStrategy: &RolloutStrategy{}, }, } kcp.Default() g.Expect(kcp.Spec.InfrastructureTemplate.Namespace).To(Equal(kcp.Namespace)) g.Expect(kcp.Spec.Version).To(Equal("v1.18.3")) + g.Expect(kcp.Spec.RolloutStrategy.Type).To(Equal(RollingUpdateStrategyType)) + g.Expect(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal).To(Equal(int32(1))) } func TestKubeadmControlPlaneValidateCreate(t *testing.T) { @@ -60,11 +64,22 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { }, Replicas: pointer.Int32Ptr(1), Version: "v1.19.0", + RolloutStrategy: &RolloutStrategy{ + Type: RollingUpdateStrategyType, + RollingUpdate: &RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, }, } invalidNamespace := valid.DeepCopy() invalidNamespace.Spec.InfrastructureTemplate.Namespace = "bar" + invalidMaxSurge := valid.DeepCopy() + invalidMaxSurge.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = int32(3) + missingReplicas := valid.DeepCopy() missingReplicas.Spec.Replicas = nil @@ -142,6 +157,11 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) { expectErr: true, kcp: invalidVersion1, }, + { + name: "should return error when maxSurge is not 1", + expectErr: true, + kcp: invalidMaxSurge, + }, } for _, tt := range tests { diff --git a/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go b/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go index be97eb20a684..7c33554eede0 100644 --- a/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go +++ b/controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go @@ -23,6 +23,7 @@ package v1alpha3 import ( "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" apiv1alpha3 "sigs.k8s.io/cluster-api/api/v1alpha3" ) @@ -104,6 +105,11 @@ func (in *KubeadmControlPlaneSpec) DeepCopyInto(out *KubeadmControlPlaneSpec) { *out = new(v1.Duration) **out = **in } + if in.RolloutStrategy != nil { + in, out := &in.RolloutStrategy, &out.RolloutStrategy + *out = new(RolloutStrategy) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeadmControlPlaneSpec. @@ -142,3 +148,43 @@ func (in *KubeadmControlPlaneStatus) DeepCopy() *KubeadmControlPlaneStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdate) DeepCopyInto(out *RollingUpdate) { + *out = *in + if in.MaxSurge != nil { + in, out := &in.MaxSurge, &out.MaxSurge + *out = new(intstr.IntOrString) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdate. +func (in *RollingUpdate) DeepCopy() *RollingUpdate { + if in == nil { + return nil + } + out := new(RollingUpdate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RolloutStrategy) DeepCopyInto(out *RolloutStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdate) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RolloutStrategy. +func (in *RolloutStrategy) DeepCopy() *RolloutStrategy { + if in == nil { + return nil + } + out := new(RolloutStrategy) + in.DeepCopyInto(out) + return out +} diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml index 6cd9780bd7b6..0e03dd417821 100644 --- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml @@ -1045,6 +1045,30 @@ spec: This is a pointer to distinguish between explicit zero and not specified. format: int32 type: integer + rolloutStrategy: + description: The RolloutStrategy to use to replace control plane machines + with new ones. + properties: + rollingUpdate: + description: Rolling update config params. Present only if RolloutStrategyType + = RollingUpdate. + properties: + maxSurge: + anyOf: + - type: integer + - type: string + description: 'The maximum number of control planes that can + be scheduled above or under the desired number of control + planes. Value can be an absolute number 1 or 0. Defaults + to 1. Example: when this is set to 1, the control plane + can be scaled up immediately when the rolling update starts.' + x-kubernetes-int-or-string: true + type: object + type: + description: Type of rollout. Currently the only supported strategy + is "RollingUpdate". Default is RollingUpdate. + type: string + type: object upgradeAfter: description: UpgradeAfter is a field to indicate an upgrade should be performed after the specified time even if no changes have been diff --git a/controlplane/kubeadm/controllers/controller_test.go b/controlplane/kubeadm/controllers/controller_test.go index 0346812432c8..e50d5f77bc22 100644 --- a/controlplane/kubeadm/controllers/controller_test.go +++ b/controlplane/kubeadm/controllers/controller_test.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "k8s.io/klog/klogr" @@ -1345,6 +1346,14 @@ func createClusterWithControlPlane() (*clusterv1.Cluster, *controlplanev1.Kubead }, Replicas: pointer.Int32Ptr(int32(3)), Version: "v1.16.6", + RolloutStrategy: &controlplanev1.RolloutStrategy{ + Type: "RollingUpdate", + RollingUpdate: &controlplanev1.RollingUpdate{ + MaxSurge: &intstr.IntOrString{ + IntVal: 1, + }, + }, + }, }, } diff --git a/controlplane/kubeadm/controllers/upgrade.go b/controlplane/kubeadm/controllers/upgrade.go index dcce8f7b75d2..edab1974171e 100644 --- a/controlplane/kubeadm/controllers/upgrade.go +++ b/controlplane/kubeadm/controllers/upgrade.go @@ -100,14 +100,17 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane( return ctrl.Result{}, errors.Wrap(err, "failed to upgrade kubelet config map") } - status, err := workloadCluster.ClusterStatus(ctx) - if err != nil { - return ctrl.Result{}, err - } - - if status.Nodes <= *kcp.Spec.Replicas { - // scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs - return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) + switch kcp.Spec.RolloutStrategy.Type { + case controlplanev1.RollingUpdateStrategyType: + // RolloutStrategy is currently defaulted and validated to be RollingUpdate + // We can ignore MaxUnavailable because we are enforcing health checks before we get here. + maxNodes := *kcp.Spec.Replicas + int32(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue()) + if int32(controlPlane.Machines.Len()) < maxNodes { + // scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs + return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane) + } + return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade) + default: + return ctrl.Result{}, errors.New("rolloutStrategy type is not set to rollingupdatestrategytype, unable to determine the strategy for rolling out machines") } - return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade) }