Skip to content

Commit

Permalink
Backport KCP Rollout Strategy to release-0.3
Browse files Browse the repository at this point in the history
  • Loading branch information
jan-est committed Mar 11, 2021
1 parent a685965 commit 5630cbc
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 4 deletions.
41 changes: 41 additions & 0 deletions controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,21 @@ package v1alpha3
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"

cabpkv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
"sigs.k8s.io/cluster-api/errors"
)

type RolloutStrategyType string

const (
// Replace the old control planes by new one using rolling update
// i.e. gradually scale up or down the old control planes and scale up or down the new one.
RollingUpdateStrategyType RolloutStrategyType = "RollingUpdate"
)

const (
KubeadmControlPlaneFinalizer = "kubeadm.controlplane.cluster.x-k8s.io"

Expand Down Expand Up @@ -72,6 +81,38 @@ type KubeadmControlPlaneSpec struct {
// NOTE: NodeDrainTimeout is different from `kubectl drain --timeout`
// +optional
NodeDrainTimeout *metav1.Duration `json:"nodeDrainTimeout,omitempty"`

// The RolloutStrategy to use to replace control plane machines with
// new ones.
// +optional
RolloutStrategy *RolloutStrategy `json:"rolloutStrategy,omitempty"`
}

// RolloutStrategy describes how to replace existing machines
// with new ones.
type RolloutStrategy struct {
// Type of rollout. Currently the only supported strategy is
// "RollingUpdate".
// Default is RollingUpdate.
// +optional
Type RolloutStrategyType `json:"type,omitempty"`

// Rolling update config params. Present only if
// RolloutStrategyType = RollingUpdate.
// +optional
RollingUpdate *RollingUpdate `json:"rollingUpdate,omitempty"`
}

// RollingUpdate is used to control the desired behavior of rolling update.
type RollingUpdate struct {
// The maximum number of control planes that can be scheduled above or under the
// desired number of control planes.
// Value can be an absolute number 1 or 0.
// Defaults to 1.
// Example: when this is set to 1, the control plane can be scaled
// up immediately when the rolling update starts.
// +optional
MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
}

// KubeadmControlPlaneStatus defines the observed state of KubeadmControlPlane.
Expand Down
56 changes: 56 additions & 0 deletions controlplane/kubeadm/api/v1alpha3/kubeadm_control_plane_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/validation/field"
kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1"
"sigs.k8s.io/cluster-api/util"
Expand Down Expand Up @@ -64,6 +65,25 @@ func (in *KubeadmControlPlane) Default() {
if !strings.HasPrefix(in.Spec.Version, "v") {
in.Spec.Version = "v" + in.Spec.Version
}

ios1 := intstr.FromInt(1)

if in.Spec.RolloutStrategy == nil {
in.Spec.RolloutStrategy = &RolloutStrategy{}
}

// Enforce RollingUpdate strategy and default MaxSurge if not set.
if in.Spec.RolloutStrategy != nil {
if len(in.Spec.RolloutStrategy.Type) == 0 {
in.Spec.RolloutStrategy.Type = RollingUpdateStrategyType
}
if in.Spec.RolloutStrategy.Type == RollingUpdateStrategyType {
if in.Spec.RolloutStrategy.RollingUpdate == nil {
in.Spec.RolloutStrategy.RollingUpdate = &RollingUpdate{}
}
in.Spec.RolloutStrategy.RollingUpdate.MaxSurge = intstr.ValueOrDefault(in.Spec.RolloutStrategy.RollingUpdate.MaxSurge, ios1)
}
}
}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
Expand Down Expand Up @@ -119,6 +139,7 @@ func (in *KubeadmControlPlane) ValidateUpdate(old runtime.Object) error {
{spec, "version"},
{spec, "upgradeAfter"},
{spec, "nodeDrainTimeout"},
{spec, "rolloutStrategy"},
}

allErrs := in.validateCommon()
Expand Down Expand Up @@ -272,6 +293,41 @@ func (in *KubeadmControlPlane) validateCommon() (allErrs field.ErrorList) {
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "version"), in.Spec.Version, "must be a valid semantic version"))
}

if in.Spec.RolloutStrategy != nil {
if in.Spec.RolloutStrategy.Type != RollingUpdateStrategyType {
allErrs = append(
allErrs,
field.Required(
field.NewPath("spec", "rolloutStrategy", "type"),
"only RollingUpdateStrategyType is supported",
),
)
}

ios1 := intstr.FromInt(1)
ios0 := intstr.FromInt(0)

if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge == ios0 && *in.Spec.Replicas < int32(3) {
allErrs = append(
allErrs,
field.Required(
field.NewPath("spec", "rolloutStrategy", "rollingUpdate"),
"when KubeadmControlPlane is configured to scale-in, replica count needs to be at least 3",
),
)
}

if *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios1 && *in.Spec.RolloutStrategy.RollingUpdate.MaxSurge != ios0 {
allErrs = append(
allErrs,
field.Required(
field.NewPath("spec", "rolloutStrategy", "rollingUpdate", "maxSurge"),
"value must be 1 or 0",
),
)
}
}

allErrs = append(allErrs, in.validateCoreDNSImage()...)

return allErrs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha3"
kubeadmv1beta1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1"
Expand All @@ -39,12 +40,15 @@ func TestKubeadmControlPlaneDefault(t *testing.T) {
Spec: KubeadmControlPlaneSpec{
InfrastructureTemplate: corev1.ObjectReference{},
Version: "1.18.3",
RolloutStrategy: &RolloutStrategy{},
},
}
kcp.Default()

g.Expect(kcp.Spec.InfrastructureTemplate.Namespace).To(Equal(kcp.Namespace))
g.Expect(kcp.Spec.Version).To(Equal("v1.18.3"))
g.Expect(kcp.Spec.RolloutStrategy.Type).To(Equal(RollingUpdateStrategyType))
g.Expect(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal).To(Equal(int32(1)))
}

func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
Expand All @@ -60,11 +64,22 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
},
Replicas: pointer.Int32Ptr(1),
Version: "v1.19.0",
RolloutStrategy: &RolloutStrategy{
Type: RollingUpdateStrategyType,
RollingUpdate: &RollingUpdate{
MaxSurge: &intstr.IntOrString{
IntVal: 1,
},
},
},
},
}
invalidNamespace := valid.DeepCopy()
invalidNamespace.Spec.InfrastructureTemplate.Namespace = "bar"

invalidMaxSurge := valid.DeepCopy()
invalidMaxSurge.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntVal = int32(3)

missingReplicas := valid.DeepCopy()
missingReplicas.Spec.Replicas = nil

Expand Down Expand Up @@ -142,6 +157,11 @@ func TestKubeadmControlPlaneValidateCreate(t *testing.T) {
expectErr: true,
kcp: invalidVersion1,
},
{
name: "should return error when maxSurge is not 1",
expectErr: true,
kcp: invalidMaxSurge,
},
}

for _, tt := range tests {
Expand Down
46 changes: 46 additions & 0 deletions controlplane/kubeadm/api/v1alpha3/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -1045,6 +1045,30 @@ spec:
This is a pointer to distinguish between explicit zero and not specified.
format: int32
type: integer
rolloutStrategy:
description: The RolloutStrategy to use to replace control plane machines
with new ones.
properties:
rollingUpdate:
description: Rolling update config params. Present only if RolloutStrategyType
= RollingUpdate.
properties:
maxSurge:
anyOf:
- type: integer
- type: string
description: 'The maximum number of control planes that can
be scheduled above or under the desired number of control
planes. Value can be an absolute number 1 or 0. Defaults
to 1. Example: when this is set to 1, the control plane
can be scaled up immediately when the rolling update starts.'
x-kubernetes-int-or-string: true
type: object
type:
description: Type of rollout. Currently the only supported strategy
is "RollingUpdate". Default is RollingUpdate.
type: string
type: object
upgradeAfter:
description: UpgradeAfter is a field to indicate an upgrade should
be performed after the specified time even if no changes have been
Expand Down
9 changes: 9 additions & 0 deletions controlplane/kubeadm/controllers/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/tools/record"
"k8s.io/klog/klogr"
Expand Down Expand Up @@ -1345,6 +1346,14 @@ func createClusterWithControlPlane() (*clusterv1.Cluster, *controlplanev1.Kubead
},
Replicas: pointer.Int32Ptr(int32(3)),
Version: "v1.16.6",
RolloutStrategy: &controlplanev1.RolloutStrategy{
Type: "RollingUpdate",
RollingUpdate: &controlplanev1.RollingUpdate{
MaxSurge: &intstr.IntOrString{
IntVal: 1,
},
},
},
},
}

Expand Down
21 changes: 17 additions & 4 deletions controlplane/kubeadm/controllers/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
) (ctrl.Result, error) {
logger := controlPlane.Logger()

if kcp.Spec.RolloutStrategy == nil && kcp.Spec.RolloutStrategy.RollingUpdate == nil {
return ctrl.Result{}, errors.New("rolloutStrategy is not set")
}

// TODO: handle reconciliation of etcd members and kubeadm config in case they get out of sync with cluster

workloadCluster, err := r.managementCluster.GetWorkloadCluster(ctx, util.ObjectKey(cluster))
Expand Down Expand Up @@ -105,9 +109,18 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
return ctrl.Result{}, err
}

if status.Nodes <= *kcp.Spec.Replicas {
// scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane)
switch kcp.Spec.RolloutStrategy.Type {
case controlplanev1.RollingUpdateStrategyType:
// RolloutStrategy is currently defaulted and validated to be RollingUpdate
// We can ignore MaxUnavailable because we are enforcing health checks before we get here.
maxNodes := *kcp.Spec.Replicas + int32(kcp.Spec.RolloutStrategy.RollingUpdate.MaxSurge.IntValue())
if status.Nodes < maxNodes {
// scaleUp ensures that we don't continue scaling up while waiting for Machines to have NodeRefs
return r.scaleUpControlPlane(ctx, cluster, kcp, controlPlane)
}
return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade)
default:
logger.Info("RolloutStrategy type is not set to RollingUpdateStrategyType, unable to determine the strategy for rolling out machines")
return ctrl.Result{}, nil
}
return r.scaleDownControlPlane(ctx, cluster, kcp, controlPlane, machinesRequireUpgrade)
}

0 comments on commit 5630cbc

Please sign in to comment.