KEP-3857: Recursive Read-only (RRO) mounts (#370)

Signed-off-by: Akihiro Suda <[email protected]>

.github/.env

@@ -1,3 +1,3 @@
-GO_VERSION=1.21.9
+GO_VERSION=1.22.4
 HUGO_VERSION=0.114.0
 LIMA_VERSION=0.20.1

core/container_create.go

@@ -72,6 +72,21 @@ func (ds *dockerService) CreateContainer(
 	if err != nil {
 		return nil, fmt.Errorf("unable to get container's sandbox ID: %v", err)
 	}
+	rtHandlers, err := ds.getRuntimeHandlers()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get container's runtime handlers: %v", err)
+	}
+	var rtHandler *v1.RuntimeHandler
+	for _, h := range rtHandlers {
+		if h.Name == sandboxInfo.HostConfig.Runtime {
+			rtHandler = h
+			break
+		}
+	}
+	mountBindings, err := libdocker.GenerateMountBindings(mounts, terminationMessagePath, rtHandler)
+	if err != nil {
+		return nil, err
+	}
 	createConfig := dockerbackend.ContainerCreateConfig{
 		Name: containerName,
 		Config: &container.Config{
@@ -92,7 +107,7 @@ func (ds *dockerService) CreateContainer(
 			},
 		},
 		HostConfig: &container.HostConfig{
-			Mounts: libdocker.GenerateMountBindings(mounts, terminationMessagePath),
+			Mounts: mountBindings,
 			RestartPolicy: container.RestartPolicy{
 				Name: "no",
 			},

core/docker_service.go

@@ -17,6 +17,7 @@ limitations under the License.
 package core
 
 import (
+	"cmp"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -26,6 +27,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"sync"
 	"time"
 
@@ -42,6 +44,7 @@ import (
 	"github.com/blang/semver"
 	dockertypes "github.com/docker/docker/api/types"
 	dockersystem "github.com/docker/docker/api/types/system"
+	ociruntimefeatures "github.com/opencontainers/runtime-spec/specs-go/features"
 	"github.com/sirupsen/logrus"
 
 	v1 "k8s.io/api/core/v1"
@@ -370,6 +373,59 @@ func (ds *dockerService) getDockerInfo() (*dockersystem.Info, error) {
 	return info, nil
 }
 
+func (ds *dockerService) getRuntimeHandlers() ([]*runtimeapi.RuntimeHandler, error) {
+	info, err := ds.getDockerInfo()
+	if err != nil {
+		return nil, err
+	}
+	handlersX, err := ds.systemInfoCache.Memoize("docker_info_handlers", systemInfoCacheMinTTL, func() (interface{}, error) {
+		return getRuntimeHandlers(info)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get runtime handlers: %v", err)
+	}
+	return handlersX.([]*runtimeapi.RuntimeHandler), nil
+}
+
+func getRuntimeHandlers(info *dockersystem.Info) ([]*runtimeapi.RuntimeHandler, error) {
+	var handlers []*runtimeapi.RuntimeHandler
+	for dockerName, dockerRT := range info.Runtimes {
+		var rro bool
+		if kernelSupportsRRO {
+			if ociFeaturesStr, ok := dockerRT.Status["org.opencontainers.runtime-spec.features"]; ok {
+				// "org.opencontainers.runtime-spec.features" status is available since Docker v25 (API v1.44)
+				var ociFeatures ociruntimefeatures.Features
+				if err := json.Unmarshal([]byte(ociFeaturesStr), &ociFeatures); err != nil {
+					return handlers, fmt.Errorf("failed to unmarshal %q: %v", ociFeaturesStr, err)
+				}
+				// "rro" mount type is supported since runc v1.1
+				rro = slices.Contains(ociFeatures.MountOptions, "rro")
+			}
+		}
+		features := &runtimeapi.RuntimeHandlerFeatures{
+			RecursiveReadOnlyMounts: rro,
+			UserNamespaces:          false, // TODO
+		}
+		handlers = append(handlers, &runtimeapi.RuntimeHandler{
+			Name:     dockerName,
+			Features: features,
+		})
+		if dockerName == info.DefaultRuntime {
+			handlers = append([]*runtimeapi.RuntimeHandler{
+				&runtimeapi.RuntimeHandler{
+					Name:     "",
+					Features: features,
+				},
+			}, handlers...)
+		}
+	}
+	// info.Runtimes is unmarshalized as a map in Go, so we cannot preserve the original ordering
+	slices.SortStableFunc(handlers, func(a, b *runtimeapi.RuntimeHandler) int {
+		return cmp.Compare(a.Name, b.Name)
+	})
+	return handlers, nil
+}
+
 // UpdateRuntimeConfig updates the runtime config. Currently only handles podCIDR updates.
 func (ds *dockerService) UpdateRuntimeConfig(
 	_ context.Context,
@@ -429,7 +485,14 @@ func (ds *dockerService) Status(
 		networkReady.Message = fmt.Sprintf("docker: network plugin is not ready: %v", err)
 	}
 	status := &runtimeapi.RuntimeStatus{Conditions: conditions}
-	resp := &runtimeapi.StatusResponse{Status: status}
+	handlers, err := ds.getRuntimeHandlers()
+	if err != nil {
+		return nil, err
+	}
+	resp := &runtimeapi.StatusResponse{
+		Status:          status,
+		RuntimeHandlers: handlers,
+	}
 	if r.Verbose {
 		image := defaultSandboxImage
 		podSandboxImage := ds.podSandboxImage

core/docker_service_linux.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2021 Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+import "github.com/docker/docker/pkg/parsers/kernel"
+
+var kernelSupportsRRO = kernel.CheckKernelVersion(5, 12, 0)

core/docker_service_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package core
 
 import (
+	"encoding/json"
 	"errors"
 	"math/rand"
 	"runtime"
@@ -28,7 +29,9 @@ import (
 
 	"github.com/blang/semver"
 	dockertypes "github.com/docker/docker/api/types"
+	dockersystem "github.com/docker/docker/api/types/system"
 	"github.com/golang/mock/gomock"
+	ociruntimefeatures "github.com/opencontainers/runtime-spec/specs-go/features"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
@@ -211,3 +214,73 @@ func TestAPIVersionWithCache(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, expected, version)
 }
+
+func TestGetRuntimeHandlers(t *testing.T) {
+	runcFeatures := ociruntimefeatures.Features{
+		MountOptions: []string{"rro"},
+	}
+	runcFeaturesJSON, err := json.Marshal(runcFeatures)
+	assert.NoError(t, err)
+	info := &dockersystem.Info{
+		Runtimes: map[string]dockersystem.RuntimeWithStatus{
+			"io.containerd.runc.v2": dockersystem.RuntimeWithStatus{
+				Runtime: dockersystem.Runtime{
+					Path: "runc",
+				},
+				Status: map[string]string{
+					"org.opencontainers.runtime-spec.features": string(runcFeaturesJSON),
+				},
+			},
+			"runc": dockersystem.RuntimeWithStatus{
+				Runtime: dockersystem.Runtime{
+					Path: "runc",
+				},
+				Status: map[string]string{
+					"org.opencontainers.runtime-spec.features": string(runcFeaturesJSON),
+				},
+			},
+			"runsc": dockersystem.RuntimeWithStatus{
+				Runtime: dockersystem.Runtime{
+					Path: "/usr/local/bin/runsc",
+				},
+			},
+		},
+		DefaultRuntime: "runc",
+	}
+
+	handlers, err := getRuntimeHandlers(info)
+	assert.NoError(t, err)
+
+	expectedHandlers := []runtimeapi.RuntimeHandler{
+		{
+			Name: "",
+			Features: &runtimeapi.RuntimeHandlerFeatures{
+				RecursiveReadOnlyMounts: true,
+			},
+		},
+		{
+			Name: "io.containerd.runc.v2",
+			Features: &runtimeapi.RuntimeHandlerFeatures{
+				RecursiveReadOnlyMounts: true,
+			},
+		},
+		{
+			Name: "runc",
+			Features: &runtimeapi.RuntimeHandlerFeatures{
+				RecursiveReadOnlyMounts: true,
+			},
+		},
+
+		{
+			Name: "runsc",
+			Features: &runtimeapi.RuntimeHandlerFeatures{
+				RecursiveReadOnlyMounts: false,
+			},
+		},
+	}
+	for i, f := range handlers {
+		assert.Equal(t, expectedHandlers[i].Name, f.Name)
+		assert.Equal(t, expectedHandlers[i].Features, f.Features)
+		// ignore protobuf fields
+	}
+}

core/docker_service_unsupported.go

@@ -0,0 +1,22 @@
+//go:build !linux
+// +build !linux
+
+/*
+Copyright 2021 Mirantis
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package core
+
+var kernelSupportsRRO = false

core/helpers_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
+	crierrors "k8s.io/cri-api/pkg/errors"
 )
 
 func TestLabelsAndAnnotationsRoundTrip(t *testing.T) {
@@ -338,11 +339,78 @@ func TestGenerateMountBindings(t *testing.T) {
 		{Type: dockermount.TypeBind, Source: "/mnt/7", Target: "/var/lib/mysql/7", BindOptions: &dockermount.BindOptions{CreateMountpoint: true}},
 		{Type: dockermount.TypeBind, Source: "/mnt/8", Target: "/var/lib/mysql/8", ReadOnly: true, BindOptions: &dockermount.BindOptions{CreateMountpoint: true, ReadOnlyNonRecursive: true, Propagation: dockermount.PropagationRShared}}, // Relabeling is not handled here
 	}
-	result := libdocker.GenerateMountBindings(mounts, "")
-
+	result, err := libdocker.GenerateMountBindings(mounts, "", nil)
+	assert.NoError(t, err)
 	assert.Equal(t, expectedResult, result)
 }
 
+func TestGenerateMountBindingsRRO(t *testing.T) {
+	handler := &runtimeapi.RuntimeHandler{
+		Name: "",
+		Features: &runtimeapi.RuntimeHandlerFeatures{
+			RecursiveReadOnlyMounts: true,
+		},
+	}
+
+	t.Run("Valid", func(t *testing.T) {
+		result, err := libdocker.GenerateMountBindings([]*runtimeapi.Mount{
+			{
+				HostPath:          "/foo",
+				ContainerPath:     "/foo",
+				Readonly:          true,
+				RecursiveReadOnly: true,
+			},
+		}, "", handler)
+		assert.NoError(t, err)
+		assert.Equal(t, []dockermount.Mount{
+			{
+				Type:        dockermount.TypeBind,
+				Source:      "/foo",
+				Target:      "/foo",
+				ReadOnly:    true,
+				BindOptions: &dockermount.BindOptions{CreateMountpoint: true, ReadOnlyNonRecursive: false},
+			},
+		}, result)
+	})
+
+	t.Run("InvalidPropagation", func(t *testing.T) {
+		_, err := libdocker.GenerateMountBindings([]*runtimeapi.Mount{
+			{
+				HostPath:          "/foo",
+				ContainerPath:     "/foo",
+				Readonly:          true,
+				RecursiveReadOnly: true,
+				Propagation:       runtimeapi.MountPropagation_PROPAGATION_BIDIRECTIONAL,
+			},
+		}, "", handler)
+		assert.ErrorContains(t, err, "recursive read-only mount needs private propagation")
+	})
+
+	t.Run("InvalidMode", func(t *testing.T) {
+		_, err := libdocker.GenerateMountBindings([]*runtimeapi.Mount{
+			{
+				HostPath:          "/foo",
+				ContainerPath:     "/foo",
+				Readonly:          false,
+				RecursiveReadOnly: true,
+			},
+		}, "", handler)
+		assert.ErrorContains(t, err, "recursive read-only mount conflicts with RW mount")
+	})
+
+	t.Run("Unsupported", func(t *testing.T) {
+		_, err := libdocker.GenerateMountBindings([]*runtimeapi.Mount{
+			{
+				HostPath:          "/foo",
+				ContainerPath:     "/foo",
+				Readonly:          true,
+				RecursiveReadOnly: true,
+			},
+		}, "", nil)
+		assert.ErrorIs(t, err, crierrors.ErrRROUnsupported)
+	})
+}
+
 func TestLimitedWriter(t *testing.T) {
 	max := func(x, y int64) int64 {
 		if x > y {