Merge pull request #1489 from rata/rata/userns-fixes

userns: Skip tests if the host doesn't support idmap mounts

pkg/validate/security_context_linux.go

@@ -18,14 +18,19 @@ package validate
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"strings"
+	"sync"
+	"syscall"
 	"time"
 
+	"golang.org/x/sys/unix"
 	internalapi "k8s.io/cri-api/pkg/apis"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 	"sigs.k8s.io/cri-tools/pkg/common"
@@ -38,6 +43,8 @@ import (
 const (
 	nginxContainerImage string = framework.DefaultRegistryE2ETestImagesPrefix + "nginx:1.14-2"
 	noNewPrivsImage     string = framework.DefaultRegistryE2ETestImagesPrefix + "nonewprivs:1.3"
+	usernsSize          int    = 65536
+	usernsHostID        int    = 65536
 )
 
 var _ = framework.KubeDescribe("Security Context", func() {
@@ -845,7 +852,12 @@ var _ = framework.KubeDescribe("Security Context", func() {
 
 	Context("UserNamespaces", func() {
 		var (
-			podName        string
+			podName string
+
+			// We call rc.Status() once and save the result in statusResp.
+			statusOnce sync.Once
+			statusResp *runtimeapi.StatusResponse
+
 			defaultMapping = []*runtimeapi.IDMapping{{
 				ContainerId: 0,
 				HostId:      1000,
@@ -858,103 +870,143 @@ var _ = framework.KubeDescribe("Security Context", func() {
 
 			// Find a working runtime handler if none provided
 			By("searching for runtime handler which supports user namespaces")
-			ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
-			defer cancel()
-			resp, err := rc.Status(ctx, false)
-			framework.ExpectNoError(err, "failed to get runtime config: %v", err)
-
-			supportsUserNamespaces := false
-			for _, rh := range resp.GetRuntimeHandlers() {
+			statusOnce.Do(func() {
+				ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+				defer cancel()
+				// Set verbose to true, other BeforeEachs calls need the info field
+				// populated.
+				// XXX: Do NOT use ":=" here, it breaks the closure reference to
+				// statusResp.
+				var err error
+				statusResp, err = rc.Status(ctx, true)
+				framework.ExpectNoError(err, "failed to get runtime config: %v", err)
+				_ = statusResp // Avoid unused variable error
+			})
+
+			var supportsUserNamespaces bool
+			for _, rh := range statusResp.GetRuntimeHandlers() {
 				if rh.GetName() == framework.TestContext.RuntimeHandler {
 					if rh.GetFeatures().GetUserNamespaces() {
 						supportsUserNamespaces = true
 						break
 					}
 				}
 			}
-
 			if !supportsUserNamespaces {
 				Skip("no runtime handler found which supports user namespaces")
 			}
 		})
 
-		It("runtime should support NamespaceMode_POD", func() {
-			namespaceOption := &runtimeapi.NamespaceOption{
-				UsernsOptions: &runtimeapi.UserNamespace{
-					Mode: runtimeapi.NamespaceMode_POD,
-					Uids: defaultMapping,
-					Gids: defaultMapping,
-				},
-			}
+		When("Host idmap mount support is needed", func() {
+			BeforeEach(func() {
+				pathIDMap := rootfsPath(statusResp.GetInfo())
+				if err := supportsIDMap(pathIDMap); err != nil {
+					Skip("ID mapping is not supported" + " with path: " + pathIDMap + ": " + err.Error())
+				}
+			})
+
+			It("runtime should support NamespaceMode_POD", func() {
+				namespaceOption := &runtimeapi.NamespaceOption{
+					UsernsOptions: &runtimeapi.UserNamespace{
+						Mode: runtimeapi.NamespaceMode_POD,
+						Uids: defaultMapping,
+						Gids: defaultMapping,
+					},
+				}
 
-			hostLogPath, podLogPath := createLogTempDir(podName)
-			defer os.RemoveAll(hostLogPath)
-			podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
-			containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
+				hostLogPath, podLogPath := createLogTempDir(podName)
+				defer os.RemoveAll(hostLogPath)
+				podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
+				containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
+
+				matchContainerOutputRe(podConfig, containerName, `\s+0\s+1000\s+100000\n`)
+			})
 
-			matchContainerOutputRe(podConfig, containerName, `\s+0\s+1000\s+100000\n`)
 		})
 
-		It("runtime should support NamespaceMode_NODE", func() {
-			namespaceOption := &runtimeapi.NamespaceOption{
-				UsernsOptions: &runtimeapi.UserNamespace{
-					Mode: runtimeapi.NamespaceMode_NODE,
-				},
-			}
+		When("Host idmap mount support is not needed", func() {
+			It("runtime should support NamespaceMode_NODE", func() {
+				namespaceOption := &runtimeapi.NamespaceOption{
+					UsernsOptions: &runtimeapi.UserNamespace{
+						Mode: runtimeapi.NamespaceMode_NODE,
+					},
+				}
 
-			hostLogPath, podLogPath := createLogTempDir(podName)
-			defer os.RemoveAll(hostLogPath)
-			podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
-			containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
+				hostLogPath, podLogPath := createLogTempDir(podName)
+				defer os.RemoveAll(hostLogPath)
+				podID, podConfig = createNamespacePodSandbox(rc, namespaceOption, podName, podLogPath)
+				containerName := runUserNamespaceContainer(rc, ic, podID, podConfig)
 
-			// 4294967295 means that the entire range is available
-			matchContainerOutputRe(podConfig, containerName, `\s+0\s+0\s+4294967295\n`)
-		})
+				// If this test is run inside a userns, we need to check the
+				// container userns is the same as the one we see outside.
+				expectedOutput := hostUsernsContent()
+				if expectedOutput == "" {
+					Fail("failed to get host userns content")
+				}
+				// The userns mapping can have several lines, we match each of them.
+				for _, line := range strings.Split(expectedOutput, "\n") {
+					if line == "" {
+						continue
+					}
+					mapping := parseUsernsMappingLine(line)
+					if len(mapping) != 3 {
+						msg := fmt.Sprintf("slice: %#v, len: %v", mapping, len(mapping))
+						Fail("Unexpected host mapping line: " + msg)
+					}
 
-		It("runtime should fail if more than one mapping provided", func() {
-			wrongMapping := []*runtimeapi.IDMapping{{
-				ContainerId: 0,
-				HostId:      1000,
-				Length:      100000,
-			}, {
-				ContainerId: 0,
-				HostId:      2000,
-				Length:      100000,
-			}}
-			usernsOptions := &runtimeapi.UserNamespace{
-				Mode: runtimeapi.NamespaceMode_POD,
-				Uids: wrongMapping,
-				Gids: wrongMapping,
-			}
+					// The container outputs the content of its /proc/self/uid_map.
+					// That output should match the regex of the host userns content.
+					containerId, hostId, length := mapping[0], mapping[1], mapping[2]
+					regex := fmt.Sprintf(`\s+%v\s+%v\s+%v`, containerId, hostId, length)
+					matchContainerOutputRe(podConfig, containerName, regex)
+				}
+			})
+
+			It("runtime should fail if more than one mapping provided", func() {
+				wrongMapping := []*runtimeapi.IDMapping{{
+					ContainerId: 0,
+					HostId:      1000,
+					Length:      100000,
+				}, {
+					ContainerId: 0,
+					HostId:      2000,
+					Length:      100000,
+				}}
+				usernsOptions := &runtimeapi.UserNamespace{
+					Mode: runtimeapi.NamespaceMode_POD,
+					Uids: wrongMapping,
+					Gids: wrongMapping,
+				}
 
-			runUserNamespacePodWithError(rc, podName, usernsOptions)
-		})
+				runUserNamespacePodWithError(rc, podName, usernsOptions)
+			})
 
-		It("runtime should fail if container ID 0 is not mapped", func() {
-			mapping := []*runtimeapi.IDMapping{{
-				ContainerId: 1,
-				HostId:      1000,
-				Length:      100000,
-			}}
-			usernsOptions := &runtimeapi.UserNamespace{
-				Mode: runtimeapi.NamespaceMode_POD,
-				Uids: mapping,
-				Gids: mapping,
-			}
+			It("runtime should fail if container ID 0 is not mapped", func() {
+				mapping := []*runtimeapi.IDMapping{{
+					ContainerId: 1,
+					HostId:      1000,
+					Length:      100000,
+				}}
+				usernsOptions := &runtimeapi.UserNamespace{
+					Mode: runtimeapi.NamespaceMode_POD,
+					Uids: mapping,
+					Gids: mapping,
+				}
 
-			runUserNamespacePodWithError(rc, podName, usernsOptions)
-		})
+				runUserNamespacePodWithError(rc, podName, usernsOptions)
+			})
 
-		It("runtime should fail with NamespaceMode_CONTAINER", func() {
-			usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_CONTAINER}
+			It("runtime should fail with NamespaceMode_CONTAINER", func() {
+				usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_CONTAINER}
 
-			runUserNamespacePodWithError(rc, podName, usernsOptions)
-		})
+				runUserNamespacePodWithError(rc, podName, usernsOptions)
+			})
 
-		It("runtime should fail with NamespaceMode_TARGET", func() {
-			usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_TARGET}
+			It("runtime should fail with NamespaceMode_TARGET", func() {
+				usernsOptions := &runtimeapi.UserNamespace{Mode: runtimeapi.NamespaceMode_TARGET}
 
-			runUserNamespacePodWithError(rc, podName, usernsOptions)
+				runUserNamespacePodWithError(rc, podName, usernsOptions)
+			})
 		})
 	})
 })
@@ -1458,3 +1510,95 @@ func runUserNamespacePodWithError(
 
 	framework.RunPodSandboxError(rc, config)
 }
+
+func supportsIDMap(path string) error {
+	treeFD, err := unix.OpenTree(-1, path, uint(unix.OPEN_TREE_CLONE|unix.OPEN_TREE_CLOEXEC))
+	if err != nil {
+		return err
+	}
+	defer unix.Close(treeFD)
+
+	// We want to test if idmap mounts are supported.
+	// So we use just some random mapping, it doesn't really matter which one.
+	// For the helper command, we just need something that is alive while we
+	// test this, a sleep 5 will do it.
+	cmd := exec.Command("sleep", "5")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Cloneflags:  syscall.CLONE_NEWUSER,
+		UidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: usernsHostID, Size: usernsSize}},
+		GidMappings: []syscall.SysProcIDMap{{ContainerID: 0, HostID: usernsHostID, Size: usernsSize}},
+	}
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	defer func() {
+		_ = cmd.Process.Kill()
+		_ = cmd.Wait()
+	}()
+
+	usernsPath := fmt.Sprintf("/proc/%d/ns/user", cmd.Process.Pid)
+	var usernsFile *os.File
+	if usernsFile, err = os.Open(usernsPath); err != nil {
+		return err
+	}
+	defer usernsFile.Close()
+
+	attr := unix.MountAttr{
+		Attr_set:  unix.MOUNT_ATTR_IDMAP,
+		Userns_fd: uint64(usernsFile.Fd()),
+	}
+	if err := unix.MountSetattr(treeFD, "", unix.AT_EMPTY_PATH, &attr); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// rootfsPath returns the parent path used for containerd stateDir (the container rootfs lives
+// inside there). If the object can't be parsed, it returns the "/var/lib".
+// Usually the rootfs is inside /var/lib and it's the same filesystem. In the end, to see if a path
+// supports idmap, we only care about its fs so this is a good fallback.
+func rootfsPath(info map[string]string) string {
+	defaultPath := "/var/lib"
+	jsonCfg, ok := info["config"]
+	if !ok {
+		return defaultPath
+	}
+
+	// Get only the StateDir from the json.
+	type containerdConfig struct {
+		StateDir string `json:"stateDir"`
+	}
+	cfg := containerdConfig{}
+	if err := json.Unmarshal([]byte(jsonCfg), &cfg); err != nil {
+		return defaultPath
+	}
+	if cfg.StateDir == "" {
+		return defaultPath
+	}
+
+	// The stateDir might have not been created yet. Let's use the parent directory that should
+	// always exist.
+	return filepath.Join(cfg.StateDir, "../")
+}
+
+func hostUsernsContent() string {
+	uidMapPath := "/proc/self/uid_map"
+	uidMapContent, err := os.ReadFile(uidMapPath)
+	if err != nil {
+		return ""
+	}
+	return string(uidMapContent)
+}
+
+func parseUsernsMappingLine(line string) []string {
+	// The line format is:
+	// <container-id> <host-id> <length>
+	// But there could be a lot of spaces between the fields.
+	line = strings.TrimSpace(line)
+	m := strings.Split(line, " ")
+	m = slices.DeleteFunc(m, func(s string) bool {
+		return s == ""
+	})
+	return m
+}