From 39e7467c047271beefa63fe6bc8a527052d2b656 Mon Sep 17 00:00:00 2001 From: networkop Date: Tue, 27 Jul 2021 08:44:44 +0100 Subject: [PATCH 01/13] Added WSL2 workaround for kubelet --- .../files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/images/base/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf b/images/base/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf index 04fd985301..ce74589adb 100644 --- a/images/base/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf +++ b/images/base/files/etc/systemd/system/kubelet.service.d/10-kubeadm.conf @@ -11,5 +11,8 @@ EnvironmentFile=-/etc/default/kubelet # On cgroup v1, the /kubelet cgroup is created in the entrypoint script before running systemd. # On cgroup v2, the /kubelet cgroup is created here. (See the comments in the entrypoint script for the reason.) ExecStartPre=/bin/sh -euc "if [ -f /sys/fs/cgroup/cgroup.controllers ]; then create-kubelet-cgroup-v2; fi" +# on WSL2 (and potentially other distros without systemd) /sys/fs/cgroup/systemd is created after the entrypoint, during /sbin/init. +# This eventually leads to kubelet failing to start, see: https://github.com/kubernetes-sigs/kind/issues/2323 +ExecStartPre=/bin/sh -euc "if [ ! -f /sys/fs/cgroup/cgroup.controllers ] && [ ! -d /sys/fs/cgroup/systemd/kubelet ]; then mkdir -p /sys/fs/cgroup/systemd/kubelet; fi" ExecStart= ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --cgroup-root=/kubelet From 8e4dd6416344d33744e7d2af2a2db68b1fdf9da4 Mon Sep 17 00:00:00 2001 From: Quique Llorente Date: Tue, 6 Jul 2021 08:34:34 +0200 Subject: [PATCH 02/13] Mount node's product files at containers When pods are running at kind cluster their product_uuid and product_name is the same since the share the kernel vfs, this PR add a new mount to OCI spec to bind mount node's product_uuid and product_name into pod's containers. Signed-off-by: Quique Llorente --- images/base/Dockerfile | 3 ++ images/base/files/etc/containerd/config.toml | 2 + .../files/usr/local/bin/mount-product-files | 43 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100755 images/base/files/usr/local/bin/mount-product-files diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 1a3df3ce39..2f04af5b32 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -111,6 +111,7 @@ RUN echo "Installing Packages ..." \ libseccomp2 pigz \ bash ca-certificates curl rsync \ nfs-common fuse-overlayfs \ + jq \ && find /lib/systemd/system/sysinit.target.wants/ -name "systemd-tmpfiles-setup.service" -delete \ && rm -f /lib/systemd/system/multi-user.target.wants/* \ && rm -f /etc/systemd/system/*.wants/* \ @@ -141,6 +142,8 @@ RUN echo "Installing containerd ..." \ && sha256sum --ignore-missing -c /tmp/runc.sha256 \ && mv /tmp/runc.${TARGETARCH} /usr/local/sbin/runc \ && chmod 755 /usr/local/sbin/runc \ + && ctr oci spec | jq '.hooks.createContainer[.hooks.createContainer| length] |= . + {"path": "/usr/local/bin/mount-product-files"}' \ + > /etc/containerd/cri-base.json \ && containerd --version \ && runc --version \ && systemctl enable containerd diff --git a/images/base/files/etc/containerd/config.toml b/images/base/files/etc/containerd/config.toml index f8cd933dd6..d62a70321d 100644 --- a/images/base/files/etc/containerd/config.toml +++ b/images/base/files/etc/containerd/config.toml @@ -17,6 +17,8 @@ version = 2 [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] # set default runtime handler to v2, which has a per-pod shim runtime_type = "io.containerd.runc.v2" + # Generated by "ctr oci spec" and modified at base container to mount poduct_uuid + base_runtime_spec = "/etc/containerd/cri-base.json" # Setup a runtime with the magic name ("test-handler") used for Kubernetes # runtime class tests ... diff --git a/images/base/files/usr/local/bin/mount-product-files b/images/base/files/usr/local/bin/mount-product-files new file mode 100755 index 0000000000..818a8572ff --- /dev/null +++ b/images/base/files/usr/local/bin/mount-product-files @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is a createContainer hook [1] that replicates the functionality from entrypoint script to mount product_name and product_uuid but from a product_name and product_uuid copied into the contianer rootfs to prevent all the containers from bind mounting the same file. Sharing the same bind mount between all the containers increases the latency accessing the container, preventing it from accessing in some cases. +# +# [1] https://github.com/opencontainers/runtime-spec/blob/master/config.md#createcontainer-hooks + +set -o errexit +set -o nounset +set -o pipefail + +# The bundle represents the dir path to container filesystem, container runtime state [1] is +# passed to the hook's stdin +# +# [1] https://github.com/opencontainers/runtime-spec/blob/master/runtime.md#state +# +bundle=$(jq -r .bundle) + +cp /kind/product_* "$bundle/rootfs/" +if [[ -f /sys/class/dmi/id/product_name ]]; then + mount -o ro,bind "$bundle/rootfs/product_name" "$bundle/rootfs/sys/class/dmi/id/product_name" +fi + +if [[ -f /sys/class/dmi/id/product_uuid ]]; then + mount -o ro,bind "$bundle/rootfs/product_uuid" "$bundle/rootfs/sys/class/dmi/id/product_uuid" +fi + +if [[ -f /sys/devices/virtual/dmi/id/product_uuid ]]; then + mount -o ro,bind "$bundle/rootfs/product_uuid" "$bundle/rootfs/sys/devices/virtual/dmi/id/product_uuid" +fi From 1e851b14de5836fd5e91161e52b80aa72c38e309 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Mon, 6 Sep 2021 18:10:19 +0800 Subject: [PATCH 03/13] upgrade pause image to 3.6 Signed-off-by: kerthcet --- images/base/files/etc/containerd/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/base/files/etc/containerd/config.toml b/images/base/files/etc/containerd/config.toml index d62a70321d..db6ee05e29 100644 --- a/images/base/files/etc/containerd/config.toml +++ b/images/base/files/etc/containerd/config.toml @@ -27,7 +27,7 @@ version = 2 [plugins."io.containerd.grpc.v1.cri"] # use fixed sandbox image - sandbox_image = "k8s.gcr.io/pause:3.5" + sandbox_image = "k8s.gcr.io/pause:3.6" # allow hugepages controller to be missing # see https://github.com/containerd/cri/pull/1501 tolerate_missing_hugepages_controller = true From 4347e13d6a181d031bc521a2050cfeda271a7bc1 Mon Sep 17 00:00:00 2001 From: guirish Date: Mon, 11 Oct 2021 11:34:58 +0000 Subject: [PATCH 04/13] Updated QEMU image and added cross build support for s390x kind binary and associated images --- hack/build/init-buildx.sh | 2 +- hack/release/build/cross.sh | 3 ++- images/Makefile.common.in | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/hack/build/init-buildx.sh b/hack/build/init-buildx.sh index 73ff2da43d..dbc58b433f 100755 --- a/hack/build/init-buildx.sh +++ b/hack/build/init-buildx.sh @@ -32,7 +32,7 @@ fi # We only need to do this setup on linux hosts if [ "$(uname)" == 'Linux' ]; then # NOTE: this is pinned to a digest for a reason! - docker run --rm --privileged tonistiigi/binfmt:qemu-v6.0.0@sha256:ce4d5a2a6ac4a189047fca2d71cbd901cc7beebacf538be95fccb3aca87cb2ec --install all + docker run --rm --privileged tonistiigi/binfmt:qemu-v6.1.0@sha256:11128304bc582dc7dbaa35947ff3e52e2610d23cecb410ddfa381a6ce74fa763 --install all fi # Ensure we use a builder that can leverage it (the default on linux will not) diff --git a/hack/release/build/cross.sh b/hack/release/build/cross.sh index ed901c19c8..f07cfba02b 100755 --- a/hack/release/build/cross.sh +++ b/hack/release/build/cross.sh @@ -46,6 +46,7 @@ export GOOS=darwin GOARCH=arm64 export GOOS=linux GOARCH=amd64 export GOOS=linux GOARCH=arm64 export GOOS=linux GOARCH=ppc64le +export GOOS=linux GOARCH=s390x EOF ) @@ -53,4 +54,4 @@ EOF cd "${REPO_ROOT}"/bin for f in kind-*; do shasum -a 256 "$f" > "$f".sha256sum; -done \ No newline at end of file +done diff --git a/images/Makefile.common.in b/images/Makefile.common.in index 973727777e..6a8a99a0b0 100644 --- a/images/Makefile.common.in +++ b/images/Makefile.common.in @@ -13,7 +13,7 @@ IMAGE?=$(REGISTRY)/$(IMAGE_NAME):$(TAG) export DOCKER_CLI_EXPERIMENTAL=enabled # build with buildx -PLATFORMS?=linux/amd64,linux/arm64 +PLATFORMS?=linux/amd64,linux/arm64,linux/s390x OUTPUT= PROGRESS=auto build: ensure-buildx @@ -35,4 +35,4 @@ quick: build ensure-buildx: ./../../hack/build/init-buildx.sh -.PHONY: push build quick ensure-buildx \ No newline at end of file +.PHONY: push build quick ensure-buildx From 9a3a9618035772069b88f58b17698fefbd915e56 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Wed, 13 Oct 2021 09:49:54 +0200 Subject: [PATCH 05/13] bump base image dependencies --- images/base/Dockerfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 2f04af5b32..d832a85ba0 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -27,32 +27,32 @@ ARG TARGETARCH # Configure containerd and runc binaries from kind-ci/containerd-nightlies repository # The repository contains latest stable releases and nightlies built for multiple architectures -ARG CONTAINERD_VERSION="1.5.5" +ARG CONTAINERD_VERSION="1.5.7" ARG CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION}" ARG CONTAINERD_URL="${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}.linux-${TARGETARCH}.tar.gz" -ARG CONTAINERD_AMD64_SHA256SUM="8130d1db8ba5a45678013465fc8dfebb252fc9bf068248c06f9dbb5d8554a5f7" -ARG CONTAINERD_ARM64_SHA256SUM="c452b52bd5ce3f788bbfbd39741c9066752803aaca5fa6da79b9658902a3a2ec" -ARG CONTAINERD_PPC64LE_SHA256SUM="28d34cec34a5df0f84d7e6b04d15b7d04be175ff1a370b8442217913cd4fb0d6" +ARG CONTAINERD_AMD64_SHA256SUM="60485e227684023da20c6eeb8353df4dbda082fde864c2898ee43d6ba98e8a9f" +ARG CONTAINERD_ARM64_SHA256SUM="106f4d77ea28c799ae72e294ba0072135f78179b1e0cada01db84aa1f92c91e4" +ARG CONTAINERD_PPC64LE_SHA256SUM="29f5175262e57e7b817b4dc84eae1e447cd5c79eef26e98b0aeccd357831d771" ARG RUNC_URL="${CONTAINERD_BASE_URL}/runc.${TARGETARCH}" -ARG RUNC_AMD64_SHA256SUM="c68e8d63cedae73df92352dcdffb3279fdb7e1d0cc823b3972c8d94ad86b9222" -ARG RUNC_ARM64_SHA256SUM="72da4b87204a955b947af939cdbeda9caeae9c33d7d7aef86bd7fdcee112dd38" -ARG RUNC_PPC64LE_SHA256SUM="b95b4dc4f4624266e9982c5c411d8cffc011d99b0c9187f689025ad41ce24c06" +ARG RUNC_AMD64_SHA256SUM="13c8d6d1d245e7897fbd0024e08a129f48e1d0fe664fde827c362f7c7243b96c" +ARG RUNC_ARM64_SHA256SUM="71523a5409dc0ef0a43715e920e630eb3801c74ddd2f1c3a7081125340d2e689" +ARG RUNC_PPC64LE_SHA256SUM="e6859ecc6c44fc3842aa5056e88b363c07bf5de1e5155f3f2b0fd6efe772abf1" # Configure crictl binary from upstream -ARG CRICTL_VERSION="v1.21.0" +ARG CRICTL_VERSION="v1.22.0" ARG CRICTL_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${TARGETARCH}.tar.gz" -ARG CRICTL_AMD64_SHA256SUM="85c78a35584971625bf1c3bcd46e5404a90396f979d7586f18b11119cb623e24" -ARG CRICTL_ARM64_SHA256SUM="454eecd29fe636282339af5b73c60234a7d10e4b11b9e18937e33056763d72cf" -ARG CRICTL_PPC64LE_SHA256SUM="0770100d30d430dbb67a58119ffed459856163ba01b6d71ac6fd4be7336253cf" +ARG CRICTL_AMD64_SHA256SUM="45e0556c42616af60ebe93bf4691056338b3ea0001c0201a6a8ff8b1dbc0652a" +ARG CRICTL_ARM64_SHA256SUM="a713c37fade0d96a989bc15ebe906e08ef5c8fe5e107c2161b0665e9963b770e" +ARG CRICTL_PPC64LE_SHA256SUM="c78bcea20c8f8ca3be0762cca7349fd2f1df520c304d0b2ef5e8fa514f64e45f" # Configure CNI binaries from upstream -ARG CNI_PLUGINS_VERSION="v0.9.1" +ARG CNI_PLUGINS_VERSION="v1.0.1" ARG CNI_PLUGINS_TARBALL="${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH}-${CNI_PLUGINS_VERSION}.tgz" ARG CNI_PLUGINS_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_TARBALL}" -ARG CNI_PLUGINS_AMD64_SHA256SUM="962100bbc4baeaaa5748cdbfce941f756b1531c2eadb290129401498bfac21e7" -ARG CNI_PLUGINS_ARM64_SHA256SUM="ef17764ffd6cdcb16d76401bac1db6acc050c9b088f1be5efa0e094ea3b01df0" -ARG CNI_PLUGINS_PPC64LE_SHA256SUM="5bd3c82ef248e5c6cc388f25545aa5a7d318778e5f9bc0a31475361bb27acefe" +ARG CNI_PLUGINS_AMD64_SHA256SUM="5238fbb2767cbf6aae736ad97a7aa29167525dcd405196dfbc064672a730d3cf" +ARG CNI_PLUGINS_ARM64_SHA256SUM="2d4528c45bdd0a8875f849a75082bc4eafe95cb61f9bcc10a6db38a031f67226" +ARG CNI_PLUGINS_PPC64LE_SHA256SUM="f078e33067e6daaef3a3a5010d6440f2464b7973dec3ca0b5d5be22fdcb1fd96" # Configure containerd-fuse-overlayfs snapshotter binary from upstream ARG CONTAINERD_FUSE_OVERLAYFS_VERSION="1.0.3" From d6255249c593c596e15d2584550cae454555b28e Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Wed, 13 Oct 2021 10:04:40 +0200 Subject: [PATCH 06/13] support s390x for base image --- images/base/Dockerfile | 10 ++++++++++ images/base/update-shasums.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/images/base/Dockerfile b/images/base/Dockerfile index d832a85ba0..1a6c85c3e0 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -33,11 +33,13 @@ ARG CONTAINERD_URL="${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION}.linu ARG CONTAINERD_AMD64_SHA256SUM="60485e227684023da20c6eeb8353df4dbda082fde864c2898ee43d6ba98e8a9f" ARG CONTAINERD_ARM64_SHA256SUM="106f4d77ea28c799ae72e294ba0072135f78179b1e0cada01db84aa1f92c91e4" ARG CONTAINERD_PPC64LE_SHA256SUM="29f5175262e57e7b817b4dc84eae1e447cd5c79eef26e98b0aeccd357831d771" +ARG CONTAINERD_S390X_SHA256SUM="4871c6622e74bf0d1e0402a351ce1d2f5b9cacf3dcfd354c4f399b67b8dfaa75" ARG RUNC_URL="${CONTAINERD_BASE_URL}/runc.${TARGETARCH}" ARG RUNC_AMD64_SHA256SUM="13c8d6d1d245e7897fbd0024e08a129f48e1d0fe664fde827c362f7c7243b96c" ARG RUNC_ARM64_SHA256SUM="71523a5409dc0ef0a43715e920e630eb3801c74ddd2f1c3a7081125340d2e689" ARG RUNC_PPC64LE_SHA256SUM="e6859ecc6c44fc3842aa5056e88b363c07bf5de1e5155f3f2b0fd6efe772abf1" +ARG RUNC_S390X_SHA256SUM="3f092321741f7a47ef5eb7f303e4365680f79970e1eebee3a9eeb0cb098f2d4f" # Configure crictl binary from upstream ARG CRICTL_VERSION="v1.22.0" @@ -45,6 +47,7 @@ ARG CRICTL_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/$ ARG CRICTL_AMD64_SHA256SUM="45e0556c42616af60ebe93bf4691056338b3ea0001c0201a6a8ff8b1dbc0652a" ARG CRICTL_ARM64_SHA256SUM="a713c37fade0d96a989bc15ebe906e08ef5c8fe5e107c2161b0665e9963b770e" ARG CRICTL_PPC64LE_SHA256SUM="c78bcea20c8f8ca3be0762cca7349fd2f1df520c304d0b2ef5e8fa514f64e45f" +ARG CRICTL_S390X_SHA256SUM="2afcf677b1c5665d0cd0f751fd5b5d7c1db6f063e007aa6b897bb5ac319611d9" # Configure CNI binaries from upstream ARG CNI_PLUGINS_VERSION="v1.0.1" @@ -53,6 +56,7 @@ ARG CNI_PLUGINS_URL="https://github.com/containernetworking/plugins/releases/dow ARG CNI_PLUGINS_AMD64_SHA256SUM="5238fbb2767cbf6aae736ad97a7aa29167525dcd405196dfbc064672a730d3cf" ARG CNI_PLUGINS_ARM64_SHA256SUM="2d4528c45bdd0a8875f849a75082bc4eafe95cb61f9bcc10a6db38a031f67226" ARG CNI_PLUGINS_PPC64LE_SHA256SUM="f078e33067e6daaef3a3a5010d6440f2464b7973dec3ca0b5d5be22fdcb1fd96" +ARG CNI_PLUGINS_S390X_SHA256SUM="468d33e16440d9ca4395c6bb2d5b71b35ae4a4df26301e4da85ac70c5ce56822" # Configure containerd-fuse-overlayfs snapshotter binary from upstream ARG CONTAINERD_FUSE_OVERLAYFS_VERSION="1.0.3" @@ -61,6 +65,7 @@ ARG CONTAINERD_FUSE_OVERLAYFS_URL="https://github.com/containerd/fuse-overlayfs- ARG CONTAINERD_FUSE_OVERLAYFS_AMD64_SHA256SUM="26c7af08d292f21e7067c0424479945bb9ff6315b49851511b2917179c5ae59a" ARG CONTAINERD_FUSE_OVERLAYFS_ARM64_SHA256SUM="68ef0896f3d5c0af73ad3d13b1b9a27f9b57cf22bdc30e36915d0f279b965bc3" ARG CONTAINERD_FUSE_OVERLAYFS_PPC64LE_SHA256SUM="49679827fa2b46dd28899bdc53c2926e83f42d305ad7ee31aeaf50dbb774a840" +ARG CONTAINERD_FUSE_OVERLAYFS_S390X_SHA256SUM="ed74e26de3215a62154b47be67953a25a15e02f7a8550408fec541d6799bc7ad" # copy in static files # all scripts are 0755: http://www.filepermissions.com/file-permission/0755 @@ -130,6 +135,7 @@ RUN echo "Installing containerd ..." \ && echo "${CONTAINERD_AMD64_SHA256SUM} /tmp/containerd.amd64.tgz" | tee /tmp/containerd.sha256 \ && echo "${CONTAINERD_ARM64_SHA256SUM} /tmp/containerd.arm64.tgz" | tee -a /tmp/containerd.sha256 \ && echo "${CONTAINERD_PPC64LE_SHA256SUM} /tmp/containerd.ppc64le.tgz" | tee -a /tmp/containerd.sha256 \ + && echo "${CONTAINERD_S390X_SHA256SUM} /tmp/containerd.s390x.tgz" | tee -a /tmp/containerd.sha256 \ && sha256sum --ignore-missing -c /tmp/containerd.sha256 \ && rm -f /tmp/containerd.sha256 \ && tar -C /usr/local -xzvf /tmp/containerd.${TARGETARCH}.tgz \ @@ -139,6 +145,7 @@ RUN echo "Installing containerd ..." \ && echo "${RUNC_AMD64_SHA256SUM} /tmp/runc.amd64" | tee /tmp/runc.sha256 \ && echo "${RUNC_ARM64_SHA256SUM} /tmp/runc.arm64" | tee -a /tmp/runc.sha256 \ && echo "${RUNC_PPC64LE_SHA256SUM} /tmp/runc.ppc64le" | tee -a /tmp/runc.sha256 \ + && echo "${RUNC_S390X_SHA256SUM} /tmp/runc.s390x" | tee -a /tmp/runc.sha256 \ && sha256sum --ignore-missing -c /tmp/runc.sha256 \ && mv /tmp/runc.${TARGETARCH} /usr/local/sbin/runc \ && chmod 755 /usr/local/sbin/runc \ @@ -153,6 +160,7 @@ RUN echo "Installing crictl ..." \ && echo "${CRICTL_AMD64_SHA256SUM} /tmp/crictl.amd64.tgz" | tee /tmp/crictl.sha256 \ && echo "${CRICTL_ARM64_SHA256SUM} /tmp/crictl.arm64.tgz" | tee -a /tmp/crictl.sha256 \ && echo "${CRICTL_PPC64LE_SHA256SUM} /tmp/crictl.ppc64le.tgz" | tee -a /tmp/crictl.sha256 \ + && echo "${CRICTL_S390X_SHA256SUM} /tmp/crictl.s390x.tgz" | tee -a /tmp/crictl.sha256 \ && sha256sum --ignore-missing -c /tmp/crictl.sha256 \ && rm -f /tmp/crictl.sha256 \ && tar -C /usr/local/bin -xzvf /tmp/crictl.${TARGETARCH}.tgz \ @@ -163,6 +171,7 @@ RUN echo "Installing CNI plugin binaries ..." \ && echo "${CNI_PLUGINS_AMD64_SHA256SUM} /tmp/cni.amd64.tgz" | tee /tmp/cni.sha256 \ && echo "${CNI_PLUGINS_ARM64_SHA256SUM} /tmp/cni.arm64.tgz" | tee -a /tmp/cni.sha256 \ && echo "${CNI_PLUGINS_PPC64LE_SHA256SUM} /tmp/cni.ppc64le.tgz" | tee -a /tmp/cni.sha256 \ + && echo "${CNI_PLUGINS_S390X_SHA256SUM} /tmp/cni.s390x.tgz" | tee -a /tmp/cni.sha256 \ && sha256sum --ignore-missing -c /tmp/cni.sha256 \ && rm -f /tmp/cni.sha256 \ && mkdir -p /opt/cni/bin \ @@ -181,6 +190,7 @@ RUN echo "Installing containerd-fuse-overlayfs ..." \ && echo "${CONTAINERD_FUSE_OVERLAYFS_AMD64_SHA256SUM} /tmp/containerd-fuse-overlayfs.amd64.tgz" | tee /tmp/containerd-fuse-overlayfs.sha256 \ && echo "${CONTAINERD_FUSE_OVERLAYFS_ARM64_SHA256SUM} /tmp/containerd-fuse-overlayfs.arm64.tgz" | tee -a /tmp/containerd-fuse-overlayfs.sha256 \ && echo "${CONTAINERD_FUSE_OVERLAYFS_PPC64LE_SHA256SUM} /tmp/containerd-fuse-overlayfs.ppc64le.tgz" | tee -a /tmp/containerd-fuse-overlayfs.sha256 \ + && echo "${CONTAINERD_FUSE_OVERLAYFS_S390X_SHA256SUM} /tmp/containerd-fuse-overlayfs.s390x.tgz" | tee -a /tmp/containerd-fuse-overlayfs.sha256 \ && sha256sum --ignore-missing -c /tmp/containerd-fuse-overlayfs.sha256 \ && rm -f /tmp/containerd-fuse-overlayfs.sha256 \ && tar -C /usr/local/bin -xzvf /tmp/containerd-fuse-overlayfs.${TARGETARCH}.tgz \ diff --git a/images/base/update-shasums.sh b/images/base/update-shasums.sh index 8876e833c2..86c30ac8b9 100755 --- a/images/base/update-shasums.sh +++ b/images/base/update-shasums.sh @@ -39,6 +39,7 @@ ARCHITECTURES=( "amd64" "arm64" "ppc64le" + "s390x" ) echo From dc200ed999993ba749e3592bd6ecbfc8bf8531f7 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 13 Oct 2021 17:49:15 +0900 Subject: [PATCH 07/13] entrypoint: allow $userns without cgroup v2 Fix issue 2490 Signed-off-by: Akihiro Suda --- images/base/files/usr/local/bin/entrypoint | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/images/base/files/usr/local/bin/entrypoint b/images/base/files/usr/local/bin/entrypoint index bf8fb7cedd..a4d7c34e7d 100755 --- a/images/base/files/usr/local/bin/entrypoint +++ b/images/base/files/usr/local/bin/entrypoint @@ -39,16 +39,14 @@ validate_userns() { echo "WARN: UserNS: expected RLIMIT_NOFILE to be at least ${nofile_hard_expected}, got ${nofile_hard}" >&2 fi - if [[ ! -f "/sys/fs/cgroup/cgroup.controllers" ]]; then - echo "ERROR: UserNS: cgroup v2 needs to be enabled" >&2 - exit 1 + if [[ -f "/sys/fs/cgroup/cgroup.controllers" ]]; then + for f in cpu memory pids; do + if ! grep -qw $f /sys/fs/cgroup/cgroup.controllers; then + echo "ERROR: UserNS: $f controller needs to be delegated" >&2 + exit 1 + fi + done fi - for f in cpu memory pids; do - if ! grep -qw $f /sys/fs/cgroup/cgroup.controllers; then - echo "ERROR: UserNS: $f controller needs to be delegated" >&2 - exit 1 - fi - done } configure_containerd() { From 941494e53f78c12d4d350e1fd9a53f931b5b32d3 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 13 Oct 2021 17:52:24 +0900 Subject: [PATCH 08/13] entrypoint: try remounting /sys even when $userns is true Co-authored-by: Felipe Santos Signed-off-by: Akihiro Suda --- images/base/files/usr/local/bin/entrypoint | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/images/base/files/usr/local/bin/entrypoint b/images/base/files/usr/local/bin/entrypoint index a4d7c34e7d..d09f0dd6b2 100755 --- a/images/base/files/usr/local/bin/entrypoint +++ b/images/base/files/usr/local/bin/entrypoint @@ -100,15 +100,19 @@ fix_mount() { sync fi - if [[ -z "${userns}" ]]; then - echo 'INFO: remounting /sys read-only' - # systemd-in-a-container should have read only /sys - # https://systemd.io/CONTAINER_INTERFACE/ - # however, we need other things from `docker run --privileged` ... - # and this flag also happens to make /sys rw, amongst other things - # - # This step is skipped when running inside UserNS, because it fails with EACCES. - mount -o remount,ro /sys + echo 'INFO: remounting /sys read-only' + # systemd-in-a-container should have read only /sys + # https://systemd.io/CONTAINER_INTERFACE/ + # however, we need other things from `docker run --privileged` ... + # and this flag also happens to make /sys rw, amongst other things + # + # This step is ignored when running inside UserNS, because it fails with EACCES. + if ! mount -o remount,ro /sys; then + if [[ -n "$userns" ]]; then + echo 'INFO: UserNS: ignoring mount fail' >&2 + else + exit 1 + fi fi echo 'INFO: making mounts shared' >&2 From d719029e27f683e9715ad031c2fd5eda84951ffb Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 13 Oct 2021 18:07:44 +0900 Subject: [PATCH 09/13] entrypoint: fix KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER was accidentally ignored when $userns is true Signed-off-by: Akihiro Suda --- images/base/files/usr/local/bin/entrypoint | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/images/base/files/usr/local/bin/entrypoint b/images/base/files/usr/local/bin/entrypoint index d09f0dd6b2..ebb8adde55 100755 --- a/images/base/files/usr/local/bin/entrypoint +++ b/images/base/files/usr/local/bin/entrypoint @@ -58,10 +58,12 @@ configure_containerd() { sed -i 's/restrict_oom_score_adj = false/restrict_oom_score_adj = true/' /etc/containerd/config.toml # Use fuse-overlayfs by default: https://github.com/kubernetes-sigs/kind/issues/2275 - snapshotter="fuse-overlayfs" + if [[ -z "$snapshotter" ]]; then + snapshotter="fuse-overlayfs" + fi else # we need to switch to the 'native' snapshotter on zfs - if [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then + if [[ -z "$snapshotter" ]] && [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then snapshotter="native" fi fi From 2d60a5efd48b6a744f5f663144a4fd912578b57c Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Wed, 13 Oct 2021 18:32:59 +0900 Subject: [PATCH 10/13] entrypoint: allow overlayfs in userns when preferrable Allow overlayfs in userns when `mount -t overlay ...` seems functional. Previously `fuse-overlayfs` was always used. `mount -t overlay` is known to work on the following environments: - Kernel >= 5.11 - Ubuntu kernel - Debian kernel with `modprobe overlay permit_mount_in_userns=1` - Sysbox However, Debian variant of overlayfs is avoided, as there is an issue on stability. See moby/moby issue 42302. Signed-off-by: Akihiro Suda --- images/base/files/usr/local/bin/entrypoint | 44 +++++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/images/base/files/usr/local/bin/entrypoint b/images/base/files/usr/local/bin/entrypoint index ebb8adde55..d864259be8 100755 --- a/images/base/files/usr/local/bin/entrypoint +++ b/images/base/files/usr/local/bin/entrypoint @@ -49,6 +49,46 @@ validate_userns() { fi } +overlayfs_preferrable() { + if [[ -z "$userns" ]]; then + # If we are outside userns, we can always assume overlayfs is preferrable + return 0 + fi + + # Debian 10 and 11 supports overlayfs in userns with a "permit_mount_in_userns" kernel patch, + # but known to be unstable, so we avoid using it https://github.com/moby/moby/issues/42302 + if [[ -e "/sys/module/overlay/parameters/permit_mounts_in_userns" ]]; then + echo "INFO: UserNS: kernel seems supporting overlayfs with permit_mounts_in_userns, but avoiding due to instability." + return 1 + fi + + # Check overlayfs availability, by attempting to mount it. + # + # Overlayfs inside userns is known to be available for the following environments: + # - Kernel >= 5.11 (but 5.11 and 5.12 have issues on SELinux hosts. Fixed in 5.13.) + # - Ubuntu kernel + # - Debian kernel (but avoided due to instability, see the /sys/module/overlay/... check above) + # - Sysbox + tmp=$(mktemp -d) + mkdir -p "${tmp}/l" "${tmp}/u" "${tmp}/w" "${tmp}/m" + if ! mount -t overlay -o lowerdir="${tmp}/l,upperdir=${tmp}/u,workdir=${tmp}/w" overlay "${tmp}/m"; then + echo "INFO: UserNS: kernel does not seem to support overlayfs." + rm -rf "${tmp}" + return 1 + fi + umount "${tmp}/m" + rm -rf "${tmp}" + + # Detect whether SELinux is Enforcing (or Permitted) by grepping /proc/self/attr/current . + # Note that we cannot use `getenforce` command here because /sys/fs/selinux is typically not mounted for containers. + if grep -q "_t:" "/proc/self/attr/current"; then + # When the kernel is before v5.13 and SELinux is enforced, fuse-overlayfs might be safer, so we print a warning (but not an error). + # https://github.com/torvalds/linux/commit/7fa2e79a6bb924fa4b2de5766dab31f0f47b5ab6 + echo "WARN: UserNS: SELinux might be Enforcing. If you see an error related to overlayfs, try setting \`KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER=fuse-overlayfs\` ." >&2 + fi + return 0 +} + configure_containerd() { local snapshotter=${KIND_EXPERIMENTAL_CONTAINERD_SNAPSHOTTER:-} if [[ -n "$userns" ]]; then @@ -57,8 +97,8 @@ configure_containerd() { # Adjust oomScoreAdj sed -i 's/restrict_oom_score_adj = false/restrict_oom_score_adj = true/' /etc/containerd/config.toml - # Use fuse-overlayfs by default: https://github.com/kubernetes-sigs/kind/issues/2275 - if [[ -z "$snapshotter" ]]; then + # Use fuse-overlayfs if overlayfs is not preferrable: https://github.com/kubernetes-sigs/kind/issues/2275 + if [[ -z "$snapshotter" ]] && ! overlayfs_preferrable; then snapshotter="fuse-overlayfs" fi else From 958fa6f78533df8b01715d3693a2acf10e52284f Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 14 Oct 2021 00:29:13 +0200 Subject: [PATCH 11/13] bump base image --- pkg/build/nodeimage/defaults.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/build/nodeimage/defaults.go b/pkg/build/nodeimage/defaults.go index 52222b2ed4..e5bcd8ecab 100644 --- a/pkg/build/nodeimage/defaults.go +++ b/pkg/build/nodeimage/defaults.go @@ -20,4 +20,4 @@ package nodeimage const DefaultImage = "kindest/node:latest" // DefaultBaseImage is the default base image used -const DefaultBaseImage = "docker.io/kindest/base:v20210825-cb7eab3f" +const DefaultBaseImage = "docker.io/kindest/base:v20211014-2d60a5ef" From 083e409f77f37e153420365d9ac5e74716c6429d Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 14 Oct 2021 09:32:50 +0200 Subject: [PATCH 12/13] support s390x node images --- pkg/build/nodeimage/build.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/build/nodeimage/build.go b/pkg/build/nodeimage/build.go index d643cad624..00b00c2294 100644 --- a/pkg/build/nodeimage/build.go +++ b/pkg/build/nodeimage/build.go @@ -74,6 +74,7 @@ func supportedArch(arch string) bool { case "amd64": case "arm64": case "ppc64le": + case "s390x": } return true } From cbaec34ba110a984e3bb3ee8342cf1cd91134539 Mon Sep 17 00:00:00 2001 From: Antonio Ojea Date: Thu, 14 Oct 2021 10:49:05 +0200 Subject: [PATCH 13/13] ignore seccomp for building node images --- pkg/build/nodeimage/buildcontext.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/build/nodeimage/buildcontext.go b/pkg/build/nodeimage/buildcontext.go index 5b409df0ce..f07ca2173b 100644 --- a/pkg/build/nodeimage/buildcontext.go +++ b/pkg/build/nodeimage/buildcontext.go @@ -360,6 +360,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) { "--entrypoint=sleep", "--name=" + id, "--platform=" + dockerBuildOsAndArch(c.arch), + "--security-opt", "seccomp=unconfined", // ignore seccomp }, []string{ "infinity", // sleep infinitely to keep the container around