Merge pull request #4294 from fabriziopandini/default-cgroupDriver-to…

…-systemd-for-1.21

⚠️ Default Kubelet cgroupDriver to systemd for Kubernetes >= 1.21

controlplane/kubeadm/internal/workload_cluster.go

@@ -34,6 +34,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3"
 	kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3"
@@ -42,16 +43,20 @@ import (
 	containerutil "sigs.k8s.io/cluster-api/util/container"
 	"sigs.k8s.io/cluster-api/util/patch"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
 )
 
 const (
 	kubeProxyKey              = "kube-proxy"
 	kubeadmConfigKey          = "kubeadm-config"
+	kubeletConfigKey          = "kubelet"
+	cgroupDriverKey           = "cgroupDriver"
 	labelNodeRoleControlPlane = "node-role.kubernetes.io/master"
 )
 
 var (
-	ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported")
+	minVerKubeletSystemdDriver = semver.MustParse("1.21.0")
+	ErrControlPlaneMinNodes    = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported")
 )
 
 // WorkloadCluster defines all behaviors necessary to upgrade kubernetes on a workload cluster
@@ -174,6 +179,42 @@ func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Ve
 		return err
 	}
 
+	// In order to avoid using two cgroup drivers on the same machine,
+	// (cgroupfs and systemd cgroup drivers), starting from
+	// 1.21 image builder is going to configure containerd for using the
+	// systemd driver, and the Kubelet configuration must be updated accordingly
+	// NOTE: It is considered safe to update the kubelet-config-1.21 ConfigMap
+	// because only new nodes using v1.21 images will pick up the change during
+	// kubeadm join.
+	if version.GE(minVerKubeletSystemdDriver) {
+		data, ok := cm.Data[kubeletConfigKey]
+		if !ok {
+			return errors.Errorf("unable to find %q key in %s", kubeletConfigKey, cm.Name)
+		}
+		kubeletConfig, err := yamlToUnstructured([]byte(data))
+		if err != nil {
+			return errors.Wrapf(err, "unable to decode kubelet ConfigMap's %q content to Unstructured object", kubeletConfigKey)
+		}
+		cgroupDriver, _, err := unstructured.NestedString(kubeletConfig.UnstructuredContent(), cgroupDriverKey)
+		if err != nil {
+			return errors.Wrapf(err, "unable to extract %q from Kubelet ConfigMap's %q", cgroupDriverKey, cm.Name)
+		}
+
+		// If the value is not already explicitly set by the user, change according to kubeadm/image builder new requirements.
+		if cgroupDriver == "" {
+			cgroupDriver = "systemd"
+
+			if err := unstructured.SetNestedField(kubeletConfig.UnstructuredContent(), cgroupDriver, cgroupDriverKey); err != nil {
+				return errors.Wrapf(err, "unable to update %q on Kubelet ConfigMap's %q", cgroupDriverKey, cm.Name)
+			}
+			updated, err := yaml.Marshal(kubeletConfig)
+			if err != nil {
+				return errors.Wrapf(err, "unable to encode Kubelet ConfigMap's %q to YAML", cm.Name)
+			}
+			cm.Data[kubeletConfigKey] = string(updated)
+		}
+	}
+
 	// Update the name to the new name
 	cm.Name = desiredKubeletConfigMapName
 	// Clear the resource version. Is this necessary since this cm is actually a DeepCopy()?

controlplane/kubeadm/internal/workload_cluster_test.go

@@ -19,6 +19,7 @@ package internal
 import (
 	"context"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
@@ -280,33 +281,74 @@ kind: ClusterStatus
 }
 
 func TestUpdateKubeletConfigMap(t *testing.T) {
-	kubeletConfig := &corev1.ConfigMap{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:            "kubelet-config-1.1",
-			Namespace:       metav1.NamespaceSystem,
-			ResourceVersion: "some-resource-version",
-		},
-	}
-
 	g := NewWithT(t)
 	scheme := runtime.NewScheme()
 	g.Expect(corev1.AddToScheme(scheme)).To(Succeed())
 	tests := []struct {
-		name      string
-		version   semver.Version
-		objs      []runtime.Object
-		expectErr bool
+		name               string
+		version            semver.Version
+		objs               []runtime.Object
+		expectErr          bool
+		expectCgroupDriver string
 	}{
 		{
-			name:      "create new config map",
-			version:   semver.Version{Major: 1, Minor: 2},
-			objs:      []runtime.Object{kubeletConfig},
-			expectErr: false,
+			name:    "create new config map",
+			version: semver.Version{Major: 1, Minor: 20},
+			objs: []runtime.Object{&corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "kubelet-config-1.19",
+					Namespace:       metav1.NamespaceSystem,
+					ResourceVersion: "some-resource-version",
+				},
+				Data: map[string]string{
+					kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" +
+						"kind: KubeletConfiguration\n",
+				},
+			}},
+			expectErr:          false,
+			expectCgroupDriver: "",
 		},
 		{
-			name:      "returns error if cannot find previous config map",
-			version:   semver.Version{Major: 1, Minor: 2},
-			expectErr: true,
+			name:    "KubeletConfig 1.21 gets the cgroupDriver set if empty",
+			version: semver.Version{Major: 1, Minor: 21},
+			objs: []runtime.Object{&corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "kubelet-config-1.20",
+					Namespace:       metav1.NamespaceSystem,
+					ResourceVersion: "some-resource-version",
+				},
+				Data: map[string]string{
+					kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" +
+						"kind: KubeletConfiguration\n",
+				},
+			}},
+			expectErr:          false,
+			expectCgroupDriver: "systemd",
+		},
+		{
+			name:    "KubeletConfig 1.21 preserves cgroupDriver if already set",
+			version: semver.Version{Major: 1, Minor: 21},
+			objs: []runtime.Object{&corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "kubelet-config-1.20",
+					Namespace:       metav1.NamespaceSystem,
+					ResourceVersion: "some-resource-version",
+				},
+				Data: map[string]string{
+					kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" +
+						"kind: KubeletConfiguration\n" +
+						"cgroupDriver: foo\n",
+				},
+			}},
+			expectErr:          false,
+			expectCgroupDriver: "foo",
+		},
+		{
+			name:               "returns error if cannot find previous config map",
+			version:            semver.Version{Major: 1, Minor: 21},
+			objs:               nil,
+			expectErr:          true,
+			expectCgroupDriver: "",
 		},
 	}
 
@@ -327,10 +369,11 @@ func TestUpdateKubeletConfigMap(t *testing.T) {
 			var actualConfig corev1.ConfigMap
 			g.Expect(w.Client.Get(
 				ctx,
-				ctrlclient.ObjectKey{Name: "kubelet-config-1.2", Namespace: metav1.NamespaceSystem},
+				ctrlclient.ObjectKey{Name: fmt.Sprintf("kubelet-config-%d.%d", tt.version.Major, tt.version.Minor), Namespace: metav1.NamespaceSystem},
 				&actualConfig,
 			)).To(Succeed())
-			g.Expect(actualConfig.ResourceVersion).ToNot(Equal(kubeletConfig.ResourceVersion))
+			g.Expect(actualConfig.ResourceVersion).ToNot(Equal("some-resource-version"))
+			g.Expect(actualConfig.Data[kubeletConfigKey]).To(ContainSubstring(tt.expectCgroupDriver))
 		})
 	}
 }