rootless: use fuse-overlayfs snapshotter by default

The snapshotter can be overridden with `$KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER`.
e.g., `KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER=overlayfs`.

Close 2275

Signed-off-by: Akihiro Suda <[email protected]>

images/base/Dockerfile

@@ -55,6 +55,14 @@ ARG CNI_PLUGINS_AMD64_SHA256SUM="962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290
 ARG CNI_PLUGINS_ARM64_SHA256SUM="ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0"
 ARG CNI_PLUGINS_PPC64LE_SHA256SUM="5bd3c82ef248e5c6cc388f25545aa5a7d318778e5f9bc0a31475361bb27acefe"
 
+# Configure containerd-fuse-overlayfs snapshotter binary from upstream
+ARG CONTAINERD_FUSE_OVERLAYFS_VERSION="1.0.2"
+ARG CONTAINERD_FUSE_OVERLAYFS_TARBALL="v${CONTAINERD_FUSE_OVERLAYFS_VERSION}/containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-linux-${TARGETARCH}.tar.gz"
+ARG CONTAINERD_FUSE_OVERLAYFS_URL="https://github.com/containerd/fuse-overlayfs-snapshotter/releases/download/${CONTAINERD_FUSE_OVERLAYFS_TARBALL}"
+ARG CONTAINERD_FUSE_OVERLAYFS_AMD64_SHA256SUM="1f1e69f71b5ea568e93e40059af1b02a377ac0966d2acd27e4cce388a27af218"
+ARG CONTAINERD_FUSE_OVERLAYFS_ARM64_SHA256SUM="7ade1a44d880b3fb8eaa3c5ff7d3890a43b777d06ec80439c9a51ae35626c83c"
+ARG CONTAINERD_FUSE_OVERLAYFS_PPC64LE_SHA256SUM="eaf9bdd3de4514546945ea93119acea2b7bfa55ced43766e20adabddd5d20978"
+
 # copy in static files
 # all scripts are 0755: http://www.filepermissions.com/file-permission/0755
 COPY --chmod=0755 files/usr/local/bin/* /usr/local/bin/
@@ -102,7 +110,7 @@ RUN echo "Installing Packages ..." \
       conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \
       libseccomp2 pigz \
       bash ca-certificates curl rsync \
-      nfs-common \
+      nfs-common fuse-overlayfs \
     && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \
     && rm -f /lib/systemd/system/multi-user.target.wants/* \
     && rm -f /etc/systemd/system/*.wants/* \
@@ -160,6 +168,15 @@ RUN echo "Installing Packages ..." \
          -o -iname loopback \
       \) \
       -delete \
+ && echo "Installing containerd-fuse-overlayfs ..." \
+    && curl -sSL --retry 5 --output /tmp/containerd-fuse-overlayfs.${TARGETARCH}.tgz "${CONTAINERD_FUSE_OVERLAYFS_URL}" \
+    && echo "${CONTAINERD_FUSE_OVERLAYFS_AMD64_SHA256SUM}  /tmp/containerd-fuse-overlayfs.amd64.tgz" | tee /tmp/containerd-fuse-overlayfs.sha256 \
+    && echo "${CONTAINERD_FUSE_OVERLAYFS_ARM64_SHA256SUM}  /tmp/containerd-fuse-overlayfs.arm64.tgz" | tee -a /tmp/containerd-fuse-overlayfs.sha256 \
+    && echo "${CONTAINERD_FUSE_OVERLAYFS_PPC64LE_SHA256SUM}  /tmp/containerd-fuse-overlayfs.ppc64le.tgz" | tee -a /tmp/containerd-fuse-overlayfs.sha256 \
+    && sha256sum --ignore-missing -c /tmp/containerd-fuse-overlayfs.sha256 \
+    && rm -f /tmp/containerd-fuse-overlayfs.sha256 \
+    && tar -C /usr/local/bin -xzvf /tmp/containerd-fuse-overlayfs.${TARGETARCH}.tgz \
+    && rm -rf /tmp/containerd-fuse-overlayfs.${TARGETARCH}.tgz \
  && echo "Ensuring /etc/kubernetes/manifests" \
     && mkdir -p /etc/kubernetes/manifests \
  && echo "Adjusting systemd-tmpfiles timer" \

images/base/files/etc/containerd/config.toml

@@ -1,6 +1,12 @@
 # explicitly use v2 config format
 version = 2
 
+[proxy_plugins]
+# fuse-overlayfs is used for rootless
+[proxy_plugins."fuse-overlayfs"]
+  type = "snapshot"
+  address = "/run/containerd-fuse-overlayfs.sock"
+
 [plugins."io.containerd.grpc.v1.cri".containerd]
   # save disk space when using a single snapshotter
   discard_unpacked_layers = true

images/base/files/etc/systemd/system/containerd-fuse-overlayfs.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=containerd fuse-overlayfs snapshotter
+PartOf=containerd.service
+
+[Service]
+ExecStart=/usr/local/bin/containerd-fuse-overlayfs-grpc /run/containerd-fuse-overlayfs.sock /var/lib/containerd-fuse-overlayfs
+Type=notify
+Restart=always
+RestartSec=1
+
+[Install]
+WantedBy=multi-user.target

images/base/files/usr/local/bin/entrypoint

@@ -76,29 +76,15 @@ fake_sysctl() {
 }
 
 configure_containerd() {
-  # we need to switch to the 'native' snapshotter on zfs
-  if [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then
-    sed -i 's/snapshotter = "overlayfs"/snapshotter = "native"/' /etc/containerd/config.toml
-  fi
-
-  # userns (rootless) configs
+  local snapshotter=${KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER:-}
   if [[ -n "$userns" ]]; then
+    # userns (rootless) configs
+
     # Adjust oomScoreAdj
     sed -i 's/restrict_oom_score_adj = false/restrict_oom_score_adj = true/' /etc/containerd/config.toml
 
-    # mounting overlayfs inside userns requires patching kernel.
-    # Ubuntu kernel is patched by default.
-    # Debian kernel is patched by default as well, but Debian needs `sudo modprobe overlay permit_mounts_in_userns=1`.
-    local tmp
-    tmp=$(mktemp -d)
-    mkdir -p "${tmp}"/{l,u,w,m}
-    if mount -t overlay overlay -o "lowerdir=${tmp}/l,upperdir=${tmp}/u,workdir=${tmp}/w" "${tmp}/m"; then
-      umount "${tmp}/m"
-    else
-      echo 'INFO: UserNS: this kernel does not support mounting overlayfs inside userns. Disabling overlayfs'
-      sed -i 's/snapshotter = "overlayfs"/snapshotter = "native"/' /etc/containerd/config.toml
-    fi
-    rm -rf "${tmp}"
+    # Use fuse-overlayfs by default: https://github.com/kubernetes-sigs/kind/issues/2275
+    snapshotter="fuse-overlayfs"
 
     # To run vanilla kubelet inside UserNS, we need to fake several unwritable sysctl to be writable.
     # Workaround until https://github.com/kubernetes/kubernetes/pull/92863 gets merged in the upstream.
@@ -108,6 +94,19 @@ configure_containerd() {
     fake_sysctl "kernel.panic_on_oops"
     fake_sysctl "kernel.keys.root_maxkeys"
     fake_sysctl "kernel.keys.root_maxbytes"
+  else
+    # we need to switch to the 'native' snapshotter on zfs
+    if [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then
+      snapshotter="native"
+    fi
+  fi
+  if [[ -n "$snapshotter" ]]; then
+    echo "INFO: changing snapshotter from \"overlayfs\" to \"$snapshotter\""
+    sed -i "s/snapshotter = \"overlayfs\"/snapshotter = \"$snapshotter\"/" /etc/containerd/config.toml
+    if [[ "$snapshotter" = "fuse-overlayfs" ]]; then
+      echo 'INFO: enabling containerd-fuse-overlayfs service'
+      systemctl enable containerd-fuse-overlayfs
+    fi
   fi
 }
 

pkg/cluster/internal/providers/docker/provision.go

@@ -243,6 +243,11 @@ func runArgsForNode(node *config.Node, clusterIPFamily config.ClusterIPFamily, n
 		"--volume", "/var",
 		// some k8s things want to read /lib/modules
 		"--volume", "/lib/modules:/lib/modules:ro",
+		// propagate KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER to the entrypoint script
+		"-e", "KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER",
+		// enable /dev/fuse explicitly for fuse-overlayfs
+		// (Rootless Docker does not automatically mount /dev/fuse with --privileged)
+		"--device", "/dev/fuse",
 	},
 		args...,
 	)

pkg/cluster/internal/providers/podman/provision.go

@@ -206,6 +206,11 @@ func runArgsForNode(node *config.Node, clusterIPFamily config.ClusterIPFamily, n
 		"--volume", fmt.Sprintf("%s:/var:suid,exec,dev", varVolume),
 		// some k8s things want to read /lib/modules
 		"--volume", "/lib/modules:/lib/modules:ro",
+		// propagate KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER to the entrypoint script
+		"-e", "KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER",
+		// enable /dev/fuse explicitly for fuse-overlayfs
+		// (Rootless Podman does not automatically mount /dev/fuse with --privileged)
+		"--device", "/dev/fuse",
 	},
 		args...,
 	)