test/e2e: pin cgroupDriver to cgroupfs

Starting with kind v0.11, kind images will have cgroup-root=/kubelet hard-coded to fix an issue with nested cgroup hierarchies. The hierarchy below /kubelet is not compatible with systemd cgroupfs (in contrast to /). Starting with kubeadm v1.21, kubeadm will default to cgroupDriver systemd. Thus, the combination of images from kind >=v0.11 and kubeadm >=v1.21 are not compatible. To solve this problem we will pin cgroupDriver to cgroupfs until kind supports the systemd hierarchy with the /kubelet cgroup-root.
kubernetes-sigs · May 13, 2021 · 42f93d8 · 42f93d8
1 parent e39f6f3
commit 42f93d8
Show file tree

Hide file tree

Showing 7 changed files with 58 additions and 12 deletions.
diff --git a/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml
@@ -66,9 +66,17 @@ spec:
     initConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"
diff --git a/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml
@@ -23,7 +23,11 @@ spec:
       joinConfiguration:
         nodeRegistration:
           criSocket: /var/run/containerd/containerd.sock
-          kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+          kubeletExtraArgs:
+            # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+            # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+            cgroup-driver: cgroupfs
+            eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # MachineDeployment object with
 # - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object

diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml
@@ -66,10 +66,17 @@ spec:
     initConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"
-
diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml
@@ -23,7 +23,11 @@ spec:
       joinConfiguration:
         nodeRegistration:
           criSocket: /var/run/containerd/containerd.sock
-          kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+          kubeletExtraArgs:
+            # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+            # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+            cgroup-driver: cgroupfs
+            eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # MachineDeployment object
 apiVersion: cluster.x-k8s.io/v1alpha4

diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml
@@ -36,4 +36,7 @@ spec:
   joinConfiguration:
     nodeRegistration:
       kubeletExtraArgs:
+        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+        cgroup-driver: cgroupfs
         eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
diff --git a/.../infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml b/.../infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml
@@ -47,11 +47,19 @@ spec:
   initConfiguration:
     nodeRegistration:
       criSocket: /var/run/containerd/containerd.sock
-      kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+      kubeletExtraArgs:
+        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+        cgroup-driver: cgroupfs
+        eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   joinConfiguration:
     nodeRegistration:
       criSocket: /var/run/containerd/containerd.sock
-      kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+      kubeletExtraArgs:
+        # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+        # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+        cgroup-driver: cgroupfs
+        eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 # cp0 Machine
 apiVersion: cluster.x-k8s.io/v1alpha4

diff --git a/test/infrastructure/docker/templates/cluster-template-development.yaml b/test/infrastructure/docker/templates/cluster-template-development.yaml
@@ -60,11 +60,19 @@ spec:
     initConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
     joinConfiguration:
       nodeRegistration:
         criSocket: /var/run/containerd/containerd.sock
-        kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+        kubeletExtraArgs:
+          # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+          # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+          cgroup-driver: cgroupfs
+          eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
   version: "${KUBERNETES_VERSION}"
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
@@ -86,7 +94,11 @@ spec:
     spec:
       joinConfiguration:
         nodeRegistration:
-          kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'}
+          kubeletExtraArgs:
+            # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd
+            # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726
+            cgroup-driver: cgroupfs
+            eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
 ---
 apiVersion: cluster.x-k8s.io/v1alpha4
 kind: MachineDeployment