Merge pull request #2495 from neolit123/1.22-pin-cgroup-driver-in-kinder

kinder: pin the CR cgroup driver to "systemd"

kinder/pkg/build/alter/alter.go

@@ -332,6 +332,11 @@ func (c *Context) alterImage(bitsInstallers []bits.Installer, bc *bits.BuildCont
 		}
 	}
 
+	log.Info("Setup CRI ...")
+	if err := alterHelper.SetupCRI(bc); err != nil {
+		return errors.Wrapf(err, "image build Failed! Failed to setup %s", runtime)
+	}
+
 	log.Info("Start CRI ...")
 	if err := alterHelper.StartCRI(bc); err != nil {
 		return errors.Wrapf(err, "image build Failed! Failed to start %s", runtime)

kinder/pkg/cri/nodes/alterhelper.go

@@ -61,6 +61,17 @@ func (h *AlterHelper) StartCRI(bc *bits.BuildContext) error {
 	return errors.Errorf("unknown cri: %s", h.cri)
 }
 
+// SetupCRI setups the container runtime.
+func (h *AlterHelper) SetupCRI(bc *bits.BuildContext) error {
+	switch h.cri {
+	case status.ContainerdRuntime:
+		return containerd.SetupRuntime(bc)
+	case status.DockerRuntime:
+		return docker.SetupRuntime(bc)
+	}
+	return errors.Errorf("unknown cri: %s", h.cri)
+}
+
 // PreLoadInitImages preload images required by kubeadm-init into the selected container runtime that exists inside a kind(er) node
 func (h *AlterHelper) PreLoadInitImages(bc *bits.BuildContext, srcFolder string) error {
 	switch h.cri {

kinder/pkg/cri/nodes/containerd/alterhelper.go

@@ -46,6 +46,18 @@ func GetAlterContainerArgs() ([]string, []string) {
 	return runArgs, runCommands
 }
 
+// SetupRuntime setups the runtime
+func SetupRuntime(bc *bits.BuildContext) error {
+	// Append the containerd settings for a systemd cgroup driver at the end of the config.toml.
+	// This assumes runc is used as the underlying runtime.
+	if err := bc.RunInContainer("bash", "-c",
+		"printf '[plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc.options]\nsystemdCgroup = true\n' >> /etc/containerd/config.toml",
+	); err != nil {
+		return errors.Wrap(err, "could not append to /etc/containerd/config.toml")
+	}
+	return nil
+}
+
 // StartRuntime starts the runtime
 func StartRuntime(bc *bits.BuildContext) error {
 	log.Info("starting containerd")

kinder/pkg/cri/nodes/docker/alterhelper.go

@@ -37,6 +37,32 @@ func GetAlterContainerArgs() ([]string, []string) {
 	return runArgs, []string{}
 }
 
+// SetupRuntime setups the runtime
+func SetupRuntime(bc *bits.BuildContext) error {
+	// Rewrite the Docker daemon config to include:
+	// - the "cri-containerd: true", which is something that already exists in kindest/base:v20190403-1ebf15f
+	// - the cgroup driver setting (systemd)
+	if err := bc.RunInContainer("bash", "-c",
+		"printf '{\"cri-containerd\": true, \"exec-opts\": [\"native.cgroupdriver=systemd\"]}\n' > /etc/docker/daemon.json",
+	); err != nil {
+		return errors.Wrap(err, "could not overwrite /etc/docker/daemon.json")
+	}
+	// Workaround from https://github.com/kubernetes/kubernetes/issues/43704#issuecomment-289484654
+	// Using the systemd driver in our rather old base image results in errors around the kubepods.slice.
+	// Write the flags --cgroups-per-qos=false --enforce-node-allocatable="" in the KUBELET_EXTRA_ARGS file.
+	//
+	// It's not possible to pass these via the KubeletConfiguration because the validation / defaulting is bogus.
+	// Empty slice gets defaulted to "pods" and non-empty slice fails for 'cgroupsPerQOS: false' -
+	// i.e. it is not possible to pass 'none' or '[]' for the 'enforceNodeAllocatable' config field.
+	// https://github.com/kubernetes/kubernetes/blob/ea0764452222146c47ec826977f49d7001b0ea8c/pkg/kubelet/apis/config/validation/validation.go#L53-L54
+	if err := bc.RunInContainer("bash", "-c",
+		"printf 'KUBELET_EXTRA_ARGS=\"--cgroups-per-qos=false --enforce-node-allocatable=\"\"\"' > /etc/default/kubelet",
+	); err != nil {
+		return errors.Wrap(err, "could not write /etc/default/kubelet")
+	}
+	return nil
+}
+
 // StartRuntime starts the runtime
 func StartRuntime(bc *bits.BuildContext) error {
 	log.Info("starting dockerd")

kinder/pkg/kubeadm/config.go

@@ -206,6 +206,9 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+# pin the cgroup driver to systemd.
+# this assumes that the CR on the node image is configured accordingly.
+cgroupDriver: "systemd"
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
@@ -302,6 +305,9 @@ evictionHard:
   nodefs.available: "0%"
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
+# pin the cgroup driver to systemd.
+# this assumes that the CR on the node image is configured accordingly.
+cgroupDriver: "systemd"
 ---
 # no-op entry that exists solely so it can be patched
 apiVersion: kubeproxy.config.k8s.io/v1alpha1