From 8fd34d7d2670be4856ae751864fe6a524090dd30 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 11 Mar 2021 14:30:00 +0100 Subject: [PATCH] default Kubelet cgroupDriver to systemd for Kubernetes >= 1.21 --- .../kubeadm/internal/workload_cluster.go | 43 +++++++++- .../kubeadm/internal/workload_cluster_test.go | 85 ++++++++++++++----- 2 files changed, 106 insertions(+), 22 deletions(-) diff --git a/controlplane/kubeadm/internal/workload_cluster.go b/controlplane/kubeadm/internal/workload_cluster.go index de874757a0cb..add63ff1944d 100644 --- a/controlplane/kubeadm/internal/workload_cluster.go +++ b/controlplane/kubeadm/internal/workload_cluster.go @@ -34,6 +34,7 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha3" kubeadmv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha3" @@ -42,16 +43,20 @@ import ( containerutil "sigs.k8s.io/cluster-api/util/container" "sigs.k8s.io/cluster-api/util/patch" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" ) const ( kubeProxyKey = "kube-proxy" kubeadmConfigKey = "kubeadm-config" + kubeletConfigKey = "kubelet" + cgroupDriverKey = "cgroupDriver" labelNodeRoleControlPlane = "node-role.kubernetes.io/master" ) var ( - ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") + minVerKubeletSystemdDriver = semver.MustParse("1.21.0") + ErrControlPlaneMinNodes = errors.New("cluster has fewer than 2 control plane nodes; removing an etcd member is not supported") ) // WorkloadCluster defines all behaviors necessary to upgrade kubernetes on a workload cluster @@ -174,6 +179,42 @@ func (w *Workload) UpdateKubeletConfigMap(ctx context.Context, version semver.Ve return err } + // In order to avoid using two cgroup drivers on the same machine, + // (cgroupfs and systemd cgroup drivers), starting from + // 1.21 image builder is going to configure containerd for using the + // systemd driver, and the Kubelet configuration must be updated accordingly + // NOTE: It is considered safe to update the kubelet-config-1.21 ConfigMap + // because only new nodes using v1.21 images will pick up the change during + // kubeadm join. + if version.GE(minVerKubeletSystemdDriver) { + data, ok := cm.Data[kubeletConfigKey] + if !ok { + return errors.Errorf("unable to find %q key in %s", kubeletConfigKey, cm.Name) + } + kubeletConfig, err := yamlToUnstructured([]byte(data)) + if err != nil { + return errors.Wrapf(err, "unable to decode kubelet ConfigMap's %q content to Unstructured object", kubeletConfigKey) + } + cgroupDriver, _, err := unstructured.NestedString(kubeletConfig.UnstructuredContent(), cgroupDriverKey) + if err != nil { + return errors.Wrapf(err, "unable to extract %q from Kubelet ConfigMap's %q", cgroupDriverKey, cm.Name) + } + + // If the value is not already explicitly set by the user, change according to kubeadm/image builder new requirements. + if cgroupDriver == "" { + cgroupDriver = "systemd" + + if err := unstructured.SetNestedField(kubeletConfig.UnstructuredContent(), cgroupDriver, cgroupDriverKey); err != nil { + return errors.Wrapf(err, "unable to update %q on Kubelet ConfigMap's %q", cgroupDriverKey, cm.Name) + } + updated, err := yaml.Marshal(kubeletConfig) + if err != nil { + return errors.Wrapf(err, "unable to encode Kubelet ConfigMap's %q to YAML", cm.Name) + } + cm.Data[kubeletConfigKey] = string(updated) + } + } + // Update the name to the new name cm.Name = desiredKubeletConfigMapName // Clear the resource version. Is this necessary since this cm is actually a DeepCopy()? diff --git a/controlplane/kubeadm/internal/workload_cluster_test.go b/controlplane/kubeadm/internal/workload_cluster_test.go index 7cd25f0645d9..aad05bbe8677 100644 --- a/controlplane/kubeadm/internal/workload_cluster_test.go +++ b/controlplane/kubeadm/internal/workload_cluster_test.go @@ -19,6 +19,7 @@ package internal import ( "context" "errors" + "fmt" "testing" "time" @@ -280,33 +281,74 @@ kind: ClusterStatus } func TestUpdateKubeletConfigMap(t *testing.T) { - kubeletConfig := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kubelet-config-1.1", - Namespace: metav1.NamespaceSystem, - ResourceVersion: "some-resource-version", - }, - } - g := NewWithT(t) scheme := runtime.NewScheme() g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) tests := []struct { - name string - version semver.Version - objs []runtime.Object - expectErr bool + name string + version semver.Version + objs []runtime.Object + expectErr bool + expectCgroupDriver string }{ { - name: "create new config map", - version: semver.Version{Major: 1, Minor: 2}, - objs: []runtime.Object{kubeletConfig}, - expectErr: false, + name: "create new config map", + version: semver.Version{Major: 1, Minor: 20}, + objs: []runtime.Object{&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubelet-config-1.19", + Namespace: metav1.NamespaceSystem, + ResourceVersion: "some-resource-version", + }, + Data: map[string]string{ + kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" + + "kind: KubeletConfiguration\n", + }, + }}, + expectErr: false, + expectCgroupDriver: "", }, { - name: "returns error if cannot find previous config map", - version: semver.Version{Major: 1, Minor: 2}, - expectErr: true, + name: "KubeletConfig 1.21 gets the cgroupDriver set if empty", + version: semver.Version{Major: 1, Minor: 21}, + objs: []runtime.Object{&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubelet-config-1.20", + Namespace: metav1.NamespaceSystem, + ResourceVersion: "some-resource-version", + }, + Data: map[string]string{ + kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" + + "kind: KubeletConfiguration\n", + }, + }}, + expectErr: false, + expectCgroupDriver: "systemd", + }, + { + name: "KubeletConfig 1.21 preserves cgroupDriver if already set", + version: semver.Version{Major: 1, Minor: 21}, + objs: []runtime.Object{&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubelet-config-1.20", + Namespace: metav1.NamespaceSystem, + ResourceVersion: "some-resource-version", + }, + Data: map[string]string{ + kubeletConfigKey: "apiVersion: kubelet.config.k8s.io/v1beta1\n" + + "kind: KubeletConfiguration\n" + + "cgroupDriver: foo\n", + }, + }}, + expectErr: false, + expectCgroupDriver: "foo", + }, + { + name: "returns error if cannot find previous config map", + version: semver.Version{Major: 1, Minor: 21}, + objs: nil, + expectErr: true, + expectCgroupDriver: "", }, } @@ -327,10 +369,11 @@ func TestUpdateKubeletConfigMap(t *testing.T) { var actualConfig corev1.ConfigMap g.Expect(w.Client.Get( ctx, - ctrlclient.ObjectKey{Name: "kubelet-config-1.2", Namespace: metav1.NamespaceSystem}, + ctrlclient.ObjectKey{Name: fmt.Sprintf("kubelet-config-%d.%d", tt.version.Major, tt.version.Minor), Namespace: metav1.NamespaceSystem}, &actualConfig, )).To(Succeed()) - g.Expect(actualConfig.ResourceVersion).ToNot(Equal(kubeletConfig.ResourceVersion)) + g.Expect(actualConfig.ResourceVersion).ToNot(Equal("some-resource-version")) + g.Expect(actualConfig.Data[kubeletConfigKey]).To(ContainSubstring(tt.expectCgroupDriver)) }) } }