Skip to content

Commit

Permalink
enable kubeadm feature flags mutation
Browse files Browse the repository at this point in the history
  • Loading branch information
@adityabhatia
adityabhatia committed Feb 15, 2024
1 parent d63021c commit a2b80c0
Show file tree
Hide file tree
Showing 11 changed files with 183 additions and 95 deletions.
3 changes: 2 additions & 1 deletion controlplane/kubeadm/internal/controllers/controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1279,7 +1279,8 @@ dns:
type: CoreDNS
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: metav1.16.1`,
kubernetesVersion: metav1.16.1
`,
},
}
g.Expect(env.Create(ctx, kubeadmCM)).To(Succeed())
Expand Down
34 changes: 19 additions & 15 deletions controlplane/kubeadm/internal/controllers/fakes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func (f *fakeManagementCluster) List(ctx context.Context, list client.ObjectList
return f.Reader.List(ctx, list, opts...)
}

func (f *fakeManagementCluster) GetWorkloadCluster(_ context.Context, _ client.ObjectKey) (internal.WorkloadCluster, error) {
func (f *fakeManagementCluster) GetWorkloadCluster(context.Context, client.ObjectKey) (internal.WorkloadCluster, error) {
return f.Workload, nil
}

Expand Down Expand Up @@ -80,65 +80,69 @@ func (f fakeWorkloadCluster) ForwardEtcdLeadership(_ context.Context, _ *cluster
return nil
}

func (f fakeWorkloadCluster) ReconcileEtcdMembers(_ context.Context, _ []string, _ semver.Version) ([]string, error) {
func (f fakeWorkloadCluster) ReconcileEtcdMembers(context.Context, []string, semver.Version) ([]string, error) {
return nil, nil
}

func (f fakeWorkloadCluster) ClusterStatus(_ context.Context) (internal.ClusterStatus, error) {
func (f fakeWorkloadCluster) ClusterStatus(context.Context) (internal.ClusterStatus, error) {
return f.Status, nil
}

func (f fakeWorkloadCluster) GetAPIServerCertificateExpiry(_ context.Context, _ *bootstrapv1.KubeadmConfig, _ string) (*time.Time, error) {
func (f fakeWorkloadCluster) GetAPIServerCertificateExpiry(context.Context, *bootstrapv1.KubeadmConfig, string) (*time.Time, error) {
return f.APIServerCertificateExpiry, nil
}

func (f fakeWorkloadCluster) AllowBootstrapTokensToGetNodes(_ context.Context) error {
func (f fakeWorkloadCluster) AllowBootstrapTokensToGetNodes(context.Context) error {
return nil
}

func (f fakeWorkloadCluster) AllowClusterAdminPermissions(_ context.Context, _ semver.Version) error {
func (f fakeWorkloadCluster) AllowClusterAdminPermissions(context.Context, semver.Version) error {
return nil
}

func (f fakeWorkloadCluster) ReconcileKubeletRBACRole(_ context.Context, _ semver.Version) error {
func (f fakeWorkloadCluster) ReconcileKubeletRBACRole(context.Context, semver.Version) error {
return nil
}

func (f fakeWorkloadCluster) ReconcileKubeletRBACBinding(_ context.Context, _ semver.Version) error {
func (f fakeWorkloadCluster) ReconcileKubeletRBACBinding(context.Context, semver.Version) error {
return nil
}

func (f fakeWorkloadCluster) UpdateKubernetesVersionInKubeadmConfigMap(_ context.Context, _ semver.Version) error {
func (f fakeWorkloadCluster) UpdateKubernetesVersionInKubeadmConfigMap(semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return nil
}

func (f fakeWorkloadCluster) UpdateEtcdVersionInKubeadmConfigMap(_ context.Context, _, _ string, _ semver.Version) error {
func (f fakeWorkloadCluster) UpdateEtcdVersionInKubeadmConfigMap(string, string) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return nil
}

func (f fakeWorkloadCluster) UpdateKubeletConfigMap(_ context.Context, _ semver.Version) error {
func (f fakeWorkloadCluster) UpdateKubeletConfigMap(context.Context, semver.Version) error {
return nil
}

func (f fakeWorkloadCluster) RemoveEtcdMemberForMachine(_ context.Context, _ *clusterv1.Machine) error {
func (f fakeWorkloadCluster) RemoveEtcdMemberForMachine(context.Context, *clusterv1.Machine) error {
return nil
}

func (f fakeWorkloadCluster) RemoveMachineFromKubeadmConfigMap(_ context.Context, _ *clusterv1.Machine, _ semver.Version) error {
func (f fakeWorkloadCluster) RemoveMachineFromKubeadmConfigMap(context.Context, *clusterv1.Machine, semver.Version) error {
return nil
}

func (f fakeWorkloadCluster) EtcdMembers(_ context.Context) ([]string, error) {
func (f fakeWorkloadCluster) EtcdMembers(context.Context) ([]string, error) {
return f.EtcdMembersResult, nil
}

func (f fakeWorkloadCluster) UpdateClusterConfiguration(context.Context, semver.Version, ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error {
return nil
}

type fakeMigrator struct {
migrateCalled bool
migrateErr error
migratedCorefile string
}

func (m *fakeMigrator) Migrate(_, _, _ string, _ bool) (string, error) {
func (m *fakeMigrator) Migrate(string, string, string, bool) (string, error) {
m.migrateCalled = true
if m.migrateErr != nil {
return "", m.migrateErr
Expand Down
44 changes: 16 additions & 28 deletions controlplane/kubeadm/internal/controllers/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/pkg/errors"
ctrl "sigs.k8s.io/controller-runtime"

bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
"sigs.k8s.io/cluster-api/util"
Expand Down Expand Up @@ -73,9 +74,8 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
return ctrl.Result{}, errors.Wrap(err, "failed to set cluster-admin ClusterRoleBinding for kubeadm")
}

if err := workloadCluster.UpdateKubernetesVersionInKubeadmConfigMap(ctx, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update the kubernetes version in the kubeadm config map")
}
kubeadmCMMutators := make([]func(*bootstrapv1.ClusterConfiguration, *[]string), 0)
kubeadmCMMutators = append(kubeadmCMMutators, workloadCluster.UpdateKubernetesVersionInKubeadmConfigMap(parsedVersion))

if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil {
// We intentionally only parse major/minor/patch so that the subsequent code
Expand All @@ -84,38 +84,26 @@ func (r *KubeadmControlPlaneReconciler) upgradeControlPlane(
if err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", controlPlane.KCP.Spec.Version)
}

// Get the imageRepository or the correct value if nothing is set and a migration is necessary.
imageRepository := internal.ImageRepositoryFromClusterConfig(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration, parsedVersionTolerant)

if err := workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(ctx, imageRepository, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update the image repository in the kubeadm config map")
}
}

if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil && controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local != nil {
meta := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ImageMeta
if err := workloadCluster.UpdateEtcdVersionInKubeadmConfigMap(ctx, meta.ImageRepository, meta.ImageTag, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update the etcd version in the kubeadm config map")
}

extraArgs := controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.Local.ExtraArgs
if err := workloadCluster.UpdateEtcdExtraArgsInKubeadmConfigMap(ctx, extraArgs, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update the etcd extra args in the kubeadm config map")
}
}

if controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration != nil {
if err := workloadCluster.UpdateAPIServerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update api server in the kubeadm config map")
}

if err := workloadCluster.UpdateControllerManagerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update controller manager in the kubeadm config map")
}
kubeadmCMMutators = append(kubeadmCMMutators,
workloadCluster.UpdateImageRepositoryInKubeadmConfigMap(imageRepository),
workloadCluster.UpdateFeatureGatesInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.FeatureGates),
workloadCluster.UpdateEtcdVersionInKubeadmConfigMap(meta.ImageRepository, meta.ImageTag),
workloadCluster.UpdateEtcdExtraArgsInKubeadmConfigMap(extraArgs),
workloadCluster.UpdateAPIServerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer),
workloadCluster.UpdateControllerManagerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager),
workloadCluster.UpdateSchedulerInKubeadmConfigMap(controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler))
}

if err := workloadCluster.UpdateSchedulerInKubeadmConfigMap(ctx, controlPlane.KCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Scheduler, parsedVersion); err != nil {
return ctrl.Result{}, errors.Wrap(err, "failed to update scheduler in the kubeadm config map")
}
// collectively update Kubeadm config map
if err = workloadCluster.UpdateClusterConfiguration(ctx, parsedVersion, kubeadmCMMutators...); err != nil {
return ctrl.Result{}, err
}

if err := workloadCluster.UpdateKubeletConfigMap(ctx, parsedVersion); err != nil {
Expand Down
3 changes: 3 additions & 0 deletions controlplane/kubeadm/internal/webhooks/kubeadm_control_plane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ const (
ntp = "ntp"
ignition = "ignition"
diskSetup = "diskSetup"
featureGates = "featureGates"
)

const minimumCertificatesExpiryDays = 7
Expand All @@ -176,6 +177,8 @@ func (webhook *KubeadmControlPlane) ValidateUpdate(_ context.Context, oldObj, ne
{spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageRepository"},
{spec, kubeadmConfigSpec, clusterConfiguration, "dns", "imageTag"},
{spec, kubeadmConfigSpec, clusterConfiguration, "imageRepository"},
{spec, kubeadmConfigSpec, clusterConfiguration, featureGates},
{spec, kubeadmConfigSpec, clusterConfiguration, featureGates, "*"},
{spec, kubeadmConfigSpec, clusterConfiguration, apiServer},
{spec, kubeadmConfigSpec, clusterConfiguration, apiServer, "*"},
{spec, kubeadmConfigSpec, clusterConfiguration, controllerManager},
Expand Down
4 changes: 2 additions & 2 deletions controlplane/kubeadm/internal/webhooks/kubeadm_control_plane_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -886,8 +886,8 @@ func TestKubeadmControlPlaneValidateUpdate(t *testing.T) {
kcp: imageRepository,
},
{
name: "should fail when making a change to the cluster config's featureGates",
expectErr: true,
name: "should succeed when making a change to the cluster config's featureGates",
expectErr: false,
before: before,
kcp: featureGates,
},
Expand Down
76 changes: 50 additions & 26 deletions controlplane/kubeadm/internal/workload_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"fmt"
"math/big"
"reflect"
"strings"
"time"

"github.com/blang/semver/v4"
Expand Down Expand Up @@ -105,13 +106,14 @@ type WorkloadCluster interface {
// Upgrade related tasks.
ReconcileKubeletRBACBinding(ctx context.Context, version semver.Version) error
ReconcileKubeletRBACRole(ctx context.Context, version semver.Version) error
UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error
UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error
UpdateEtcdVersionInKubeadmConfigMap(ctx context.Context, imageRepository, imageTag string, version semver.Version) error
UpdateEtcdExtraArgsInKubeadmConfigMap(ctx context.Context, extraArgs map[string]string, version semver.Version) error
UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error
UpdateControllerManagerInKubeadmConfigMap(ctx context.Context, controllerManager bootstrapv1.ControlPlaneComponent, version semver.Version) error
UpdateSchedulerInKubeadmConfigMap(ctx context.Context, scheduler bootstrapv1.ControlPlaneComponent, version semver.Version) error
UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateEtcdVersionInKubeadmConfigMap(imageRepository, imageTag string) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateEtcdExtraArgsInKubeadmConfigMap(extraArgs map[string]string) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateControllerManagerInKubeadmConfigMap(controllerManager bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateSchedulerInKubeadmConfigMap(scheduler bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string)
UpdateKubeletConfigMap(ctx context.Context, version semver.Version) error
UpdateKubeProxyImageInfo(ctx context.Context, kcp *controlplanev1.KubeadmControlPlane, version semver.Version) error
UpdateCoreDNS(ctx context.Context, kcp *controlplanev1.KubeadmControlPlane, version semver.Version) error
Expand All @@ -121,6 +123,7 @@ type WorkloadCluster interface {
ForwardEtcdLeadership(ctx context.Context, machine *clusterv1.Machine, leaderCandidate *clusterv1.Machine) error
AllowBootstrapTokensToGetNodes(ctx context.Context) error
AllowClusterAdminPermissions(ctx context.Context, version semver.Version) error
UpdateClusterConfiguration(ctx context.Context, version semver.Version, mutators ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error

// State recovery tasks.
ReconcileEtcdMembers(ctx context.Context, nodeNames []string, version semver.Version) ([]string, error)
Expand Down Expand Up @@ -173,20 +176,35 @@ func (w *Workload) getConfigMap(ctx context.Context, configMap ctrlclient.Object
}

// UpdateImageRepositoryInKubeadmConfigMap updates the image repository in the kubeadm config map.
func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(ctx context.Context, imageRepository string, version semver.Version) error {
return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
func (w *Workload) UpdateImageRepositoryInKubeadmConfigMap(imageRepository string) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
if imageRepository == "" {
return
}

c.ImageRepository = imageRepository
}, version)
*updatedKeys = append(*updatedKeys, "imageRepository")
}
}

// UpdateFeatureGatesInKubeadmConfigMap updates the feature gates in the kubeadm config map.
func (w *Workload) UpdateFeatureGatesInKubeadmConfigMap(featureGates map[string]bool) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
if featureGates == nil {
return
}

c.FeatureGates = featureGates
*updatedKeys = append(*updatedKeys, "featureGates")
}
}

// UpdateKubernetesVersionInKubeadmConfigMap updates the kubernetes version in the kubeadm config map.
func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(ctx context.Context, version semver.Version) error {
return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
func (w *Workload) UpdateKubernetesVersionInKubeadmConfigMap(version semver.Version) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
c.KubernetesVersion = fmt.Sprintf("v%s", version.String())
}, version)
*updatedKeys = append(*updatedKeys, "kubernetesVersion")
}
}

// UpdateKubeletConfigMap will create a new kubelet-config-1.x config map for a new version of the kubelet.
Expand Down Expand Up @@ -270,24 +288,27 @@ func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Ve
}

// UpdateAPIServerInKubeadmConfigMap updates api server configuration in kubeadm config map.
func (w *Workload) UpdateAPIServerInKubeadmConfigMap(ctx context.Context, apiServer bootstrapv1.APIServer, version semver.Version) error {
return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
func (w *Workload) UpdateAPIServerInKubeadmConfigMap(apiServer bootstrapv1.APIServer) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
c.APIServer = apiServer
}, version)
*updatedKeys = append(*updatedKeys, "apiServer")
}
}

// UpdateControllerManagerInKubeadmConfigMap updates controller manager configuration in kubeadm config map.
func (w *Workload) UpdateControllerManagerInKubeadmConfigMap(ctx context.Context, controllerManager bootstrapv1.ControlPlaneComponent, version semver.Version) error {
return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
func (w *Workload) UpdateControllerManagerInKubeadmConfigMap(controllerManager bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
c.ControllerManager = controllerManager
}, version)
*updatedKeys = append(*updatedKeys, "controllerManager")
}
}

// UpdateSchedulerInKubeadmConfigMap updates scheduler configuration in kubeadm config map.
func (w *Workload) UpdateSchedulerInKubeadmConfigMap(ctx context.Context, scheduler bootstrapv1.ControlPlaneComponent, version semver.Version) error {
return w.updateClusterConfiguration(ctx, func(c *bootstrapv1.ClusterConfiguration) {
func (w *Workload) UpdateSchedulerInKubeadmConfigMap(scheduler bootstrapv1.ControlPlaneComponent) func(*bootstrapv1.ClusterConfiguration, *[]string) {
return func(c *bootstrapv1.ClusterConfiguration, updatedKeys *[]string) {
c.Scheduler = scheduler
}, version)
*updatedKeys = append(*updatedKeys, "scheduler")
}
}

// RemoveMachineFromKubeadmConfigMap removes the entry for the machine from the kubeadm configmap.
Expand Down Expand Up @@ -350,11 +371,11 @@ func (w *Workload) updateClusterStatus(ctx context.Context, mutator func(status
})
}

// updateClusterConfiguration gets the ClusterConfiguration kubeadm-config ConfigMap, converts it to the
// UpdateClusterConfiguration gets the ClusterConfiguration kubeadm-config ConfigMap, converts it to the
// Cluster API representation, and then applies a mutation func; if changes are detected, the
// data are converted back into the Kubeadm API version in use for the target Kubernetes version and the
// kubeadm-config ConfigMap updated.
func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func(*bootstrapv1.ClusterConfiguration), version semver.Version) error {
func (w *Workload) UpdateClusterConfiguration(ctx context.Context, version semver.Version, mutators ...func(*bootstrapv1.ClusterConfiguration, *[]string)) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
key := ctrlclient.ObjectKey{Name: kubeadmConfigKey, Namespace: metav1.NamespaceSystem}
configMap, err := w.getConfigMap(ctx, key)
Expand All @@ -373,7 +394,10 @@ func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func(
}

updatedObj := currentObj.DeepCopy()
mutator(updatedObj)
updatedKeys := make([]string, 0)
for i := range mutators {
mutators[i](updatedObj, &updatedKeys)
}

if !reflect.DeepEqual(currentObj, updatedObj) {
updatedData, err := kubeadmtypes.MarshalClusterConfigurationForVersion(updatedObj, version)
Expand All @@ -382,7 +406,7 @@ func (w *Workload) updateClusterConfiguration(ctx context.Context, mutator func(
}
configMap.Data[clusterConfigurationKey] = updatedData
if err := w.Client.Update(ctx, configMap); err != nil {
return errors.Wrap(err, "failed to upgrade the kubeadmConfigMap")
return errors.Wrapf(err, "failed to upgrade the kubeadmConfigMap with the following properties %s", strings.Join(updatedKeys, ","))
}
}
return nil
Expand Down
Loading

0 comments on commit a2b80c0

Please sign in to comment.