From 42f93d81700451cd34a3afd2f95ec6fad233fc28 Mon Sep 17 00:00:00 2001 From: Stefan Bueringer Date: Thu, 13 May 2021 18:52:22 +0200 Subject: [PATCH] test/e2e: pin cgroupDriver to cgroupfs Starting with kind v0.11, kind images will have cgroup-root=/kubelet hard-coded to fix an issue with nested cgroup hierarchies. The hierarchy below /kubelet is not compatible with systemd cgroupfs (in contrast to /). Starting with kubeadm v1.21, kubeadm will default to cgroupDriver systemd. Thus, the combination of images from kind >=v0.11 and kubeadm >=v1.21 are not compatible. To solve this problem we will pin cgroupDriver to cgroupfs until kind supports the systemd hierarchy with the /kubelet cgroup-root. --- .../v1alpha3/bases/cluster-with-kcp.yaml | 12 ++++++++++-- .../v1alpha3/bases/md.yaml | 6 +++++- .../v1alpha4/bases/cluster-with-kcp.yaml | 13 ++++++++++--- .../v1alpha4/bases/md.yaml | 6 +++++- .../v1alpha4/bases/mp.yaml | 3 +++ .../step1/cluster-with-cp0.yaml | 12 ++++++++++-- .../cluster-template-development.yaml | 18 +++++++++++++++--- 7 files changed, 58 insertions(+), 12 deletions(-) diff --git a/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml index 24f099f48f1e..b6243c954bde 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha3/bases/cluster-with-kcp.yaml @@ -66,9 +66,17 @@ spec: initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' version: "${KUBERNETES_VERSION}" diff --git a/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml index 5cef10112881..cc4de57a1890 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha3/bases/md.yaml @@ -23,7 +23,11 @@ spec: joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' --- # MachineDeployment object with # - the label nodepool=pool1 that applies to all the machines, so those machine can be targeted by the MachineHealthCheck object diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml index f6bc6be03570..0d15fb6d9e85 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha4/bases/cluster-with-kcp.yaml @@ -66,10 +66,17 @@ spec: initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' version: "${KUBERNETES_VERSION}" - \ No newline at end of file diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml index 23584de02793..343bf5783724 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha4/bases/md.yaml @@ -23,7 +23,11 @@ spec: joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' --- # MachineDeployment object apiVersion: cluster.x-k8s.io/v1alpha4 diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml index b2229d78d184..8cd739f6a9cc 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha4/bases/mp.yaml @@ -36,4 +36,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% diff --git a/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml b/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml index fbbfac3d1007..03c82d79f508 100644 --- a/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml +++ b/test/e2e/data/infrastructure-docker/v1alpha4/cluster-template-kcp-adoption/step1/cluster-with-cp0.yaml @@ -47,11 +47,19 @@ spec: initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' --- # cp0 Machine apiVersion: cluster.x-k8s.io/v1alpha4 diff --git a/test/infrastructure/docker/templates/cluster-template-development.yaml b/test/infrastructure/docker/templates/cluster-template-development.yaml index fc62dc5eb31b..55d50aca7644 100644 --- a/test/infrastructure/docker/templates/cluster-template-development.yaml +++ b/test/infrastructure/docker/templates/cluster-template-development.yaml @@ -60,11 +60,19 @@ spec: initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' joinConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4 @@ -86,7 +94,11 @@ spec: spec: joinConfiguration: nodeRegistration: - kubeletExtraArgs: {eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'} + kubeletExtraArgs: + # We have to pin the cgroupDriver to cgroupfs as kubeadm >=1.21 defaults to systemd + # kind will implement systemd support in: https://github.com/kubernetes-sigs/kind/issues/1726 + cgroup-driver: cgroupfs + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' --- apiVersion: cluster.x-k8s.io/v1alpha4 kind: MachineDeployment