Merge pull request #1818 from BenTheElder/zfs-stash

rework containerd config

images/base/files/etc/containerd/config.toml

@@ -0,0 +1,20 @@
+# explicitly use v2 config format
+version = 2
+
+# set default runtime handler to v2, which has a per-pod shim
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  default_runtime_name = "runc"
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+# Setup a runtime with the magic name ("test-handler") used for Kubernetes
+# runtime class tests ...
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.test-handler]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  # use fixed sandbox image
+  sandbox_image = "k8s.gcr.io/pause:3.3"
+  # allow hugepages controller to be missing
+  # see https://github.com/containerd/cri/pull/1501
+  tolerate_missing_hugepages_controller = true

images/base/files/usr/local/bin/entrypoint

@@ -18,6 +18,23 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+configure_containerd() {
+  # we need to switch to the 'native' snapshotter on zfs
+  if [[ "$(stat -f -c %T /kind)" == 'zfs' ]]; then
+    sed -i 's/snapshotter = "overlayfs"/snapshotter = "native"/' /etc/containerd/config.toml
+  fi
+}
+
+configure_proxy() {
+  # ensure all processes receive the proxy settings by default
+  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
+  mkdir -p /etc/systemd/system.conf.d/
+  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
+[Manager]
+DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
+EOF
+}
+
 fix_mount() {
   echo 'INFO: ensuring we can execute mount/umount even with userns-remap' 
   # necessary only when userns-remap is enabled on the host, but harmless
@@ -28,7 +45,7 @@ fix_mount() {
   # This is a workaround to an AUFS bug that might cause `Text file
   # busy` on `mount` command below. See more details in
   # https://github.com/moby/moby/issues/9547
-  if [[ "$(stat -f -c %T /bin/mount)" == 'aufs' ]]; then
+  if [[ "$(stat -f -c %T "$(which mount)")" == 'aufs' ]]; then
     echo 'INFO: detected aufs, calling sync' >&2
     sync
   fi
@@ -138,16 +155,6 @@ fix_kmsg() {
   fi
 }
 
-configure_proxy() {
-  # ensure all processes receive the proxy settings by default
-  # https://www.freedesktop.org/software/systemd/man/systemd-system.conf.html
-  mkdir -p /etc/systemd/system.conf.d/
-  cat <<EOF >/etc/systemd/system.conf.d/proxy-default-environment.conf
-[Manager]
-DefaultEnvironment="HTTP_PROXY=${HTTP_PROXY:-}" "HTTPS_PROXY=${HTTPS_PROXY:-}" "NO_PROXY=${NO_PROXY:-}"
-EOF
-}
-
 select_iptables() {
   # based on: https://github.com/kubernetes/kubernetes/blob/ffe93b3979486feb41a0f85191bdd189cbd56ccc/build/debian-iptables/iptables-wrapper
   local mode=nft
@@ -233,13 +240,15 @@ enable_network_magic(){
 }
 
 # run pre-init fixups
+# NOTE: it's important that we do configure* first in this order to avoid races
+configure_containerd
+configure_proxy
 fix_kmsg
 fix_mount
 fix_cgroup
 fix_machine_id
 fix_product_name
 fix_product_uuid
-configure_proxy
 select_iptables
 enable_network_magic
 

pkg/build/nodeimage/build_impl.go

@@ -114,28 +114,11 @@ func (c *buildContext) buildImage(bits kube.Bits) error {
 	}()
 
 	// pre-pull images that were not part of the build
-	images, err := c.prePullImages(bits, dir, containerID)
-	if err != nil {
+	if _, err = c.prePullImages(bits, dir, containerID); err != nil {
 		c.logger.Errorf("Image build Failed! Failed to pull Images: %v", err)
 		return err
 	}
 
-	// find the pause image and inject containerd config
-	pauseImage := findSandboxImage(images)
-	if pauseImage == "" {
-		return errors.New("failed to find imported pause image")
-	}
-	containerdConfig, err := getContainerdConfig(containerdConfigTemplateData{
-		SandboxImage: pauseImage,
-	})
-	if err != nil {
-		return err
-	}
-	const containerdConfigPath = "/etc/containerd/config.toml"
-	if err := createFile(cmder, containerdConfigPath, containerdConfig); err != nil {
-		return err
-	}
-
 	// Save the image changes to a new image
 	cmd := exec.Command(
 		"docker", "commit",
@@ -230,6 +213,24 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		return nil, err
 	}
 
+	// replace pause image with our own
+	config, err := exec.Output(cmder.Command("cat", "/etc/containerd/config.toml"))
+	if err != nil {
+		return nil, err
+	}
+	pauseImage, err := findSandboxImage(string(config))
+	if err != nil {
+		return nil, err
+	}
+	n := 0
+	for _, image := range requiredImages {
+		if !strings.Contains(image, "pause") {
+			requiredImages[n] = image
+			n++
+		}
+	}
+	requiredImages = append(requiredImages[:n], pauseImage)
+
 	// write the default CNI manifest
 	if err := createFile(cmder, defaultCNIManifestLocation, defaultCNIManifest); err != nil {
 		c.logger.Errorf("Image build Failed! Failed write default CNI Manifest: %v", err)

pkg/build/nodeimage/defaults.go

@@ -20,7 +20,7 @@ package nodeimage
 const DefaultImage = "kindest/node:latest"
 
 // DefaultBaseImage is the default base image used
-const DefaultBaseImage = "kindest/base:v20200826-c89bba37"
+const DefaultBaseImage = "kindest/base:v20200826-5c3ff118"
 
 // DefaultMode is the default kubernetes build mode for the built image
 // see pkg/build/kube.Bits

pkg/build/nodeimage/helpers.go

@@ -18,8 +18,10 @@ package nodeimage
 
 import (
 	"path"
+	"regexp"
 	"strings"
 
+	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 )
 
@@ -40,13 +42,10 @@ func createFile(containerCmder exec.Cmder, filePath, contents string) error {
 	).Run()
 }
 
-func findSandboxImage(images []string) string {
-	for _, image := range images {
-		// yep this seems legit
-		// https://github.com/kubernetes-sigs/kind/issues/1471#issuecomment-617579803
-		if strings.Contains(image, "pause") {
-			return image
-		}
+func findSandboxImage(config string) (string, error) {
+	match := regexp.MustCompile(`sandbox_image\s+=\s+"([^\n]+)"`).FindStringSubmatch(config)
+	if len(match) < 2 {
+		return "", errors.New("failed to parse sandbox_image from config")
 	}
-	return ""
+	return match[1], nil
 }