[v0.10][Moby LTS] Backport 4600 4602 4603 4604

Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile-upstream:master
 
-ARG RUNC_VERSION=v1.0.2
-ARG CONTAINERD_VERSION=v1.6.2
+ARG RUNC_VERSION=v1.1.12
+ARG CONTAINERD_VERSION=v1.6.28
 # containerd v1.5 for integration tests
 ARG CONTAINERD_ALT_VERSION_15=v1.5.11
 # containerd v1.4 for integration tests

client/build_test.go

@@ -63,7 +63,8 @@ func TestClientGatewayIntegration(t *testing.T) {
 	), integration.WithMirroredImages(integration.OfficialImages("busybox:latest")))
 
 	integration.Run(t, integration.TestFuncs(
-		testClientGatewayContainerSecurityMode,
+		testClientGatewayContainerSecurityModeCaps,
+		testClientGatewayContainerSecurityModeValidation,
 	), integration.WithMirroredImages(integration.OfficialImages("busybox:latest")),
 		integration.WithMatrix("secmode", map[string]interface{}{
 			"sandbox":  securitySandbox,
@@ -72,7 +73,8 @@ func TestClientGatewayIntegration(t *testing.T) {
 	)
 
 	integration.Run(t, integration.TestFuncs(
-		testClientGatewayContainerHostNetworking,
+		testClientGatewayContainerHostNetworkingAccess,
+		testClientGatewayContainerHostNetworkingValidation,
 	),
 		integration.WithMirroredImages(integration.OfficialImages("busybox:latest")),
 		integration.WithMatrix("netmode", map[string]interface{}{
@@ -1621,9 +1623,17 @@ func testClientGatewayExecFileActionError(t *testing.T, sb integration.Sandbox)
 	checkAllReleasable(t, c, sb, true)
 }
 
-// testClientGatewayContainerSecurityMode ensures that the correct security mode
+// testClientGatewayContainerSecurityModeCaps ensures that the correct security mode
 // is propagated to the gateway container
-func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox) {
+func testClientGatewayContainerSecurityModeCaps(t *testing.T, sb integration.Sandbox) {
+	testClientGatewayContainerSecurityMode(t, sb, false)
+}
+
+func testClientGatewayContainerSecurityModeValidation(t *testing.T, sb integration.Sandbox) {
+	testClientGatewayContainerSecurityMode(t, sb, true)
+}
+
+func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox, expectFail bool) {
 	requiresLinux(t)
 
 	ctx := sb.Context()
@@ -1649,6 +1659,9 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 			require.EqualValues(t, 0xa80425fb, caps)
 		}
 		allowedEntitlements = []entitlements.Entitlement{}
+		if expectFail {
+			return
+		}
 	} else {
 		integration.SkipIfDockerd(t, sb)
 		assertCaps = func(caps uint64) {
@@ -1666,6 +1679,9 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 		}
 		mode = llb.SecurityModeInsecure
 		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementSecurityInsecure}
+		if expectFail {
+			allowedEntitlements = []entitlements.Entitlement{}
+		}
 	}
 
 	b := func(ctx context.Context, c client.Client) (*client.Result, error) {
@@ -1715,6 +1731,12 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 		t.Logf("Stdout: %q", stdout.String())
 		t.Logf("Stderr: %q", stderr.String())
 
+		if expectFail {
+			require.Error(t, err)
+			require.Contains(t, err.Error(), "security.insecure is not allowed")
+			return nil, err
+		}
+
 		require.NoError(t, err)
 
 		capsValue, err := strconv.ParseUint(strings.TrimSpace(stdout.String()), 16, 64)
@@ -1729,7 +1751,13 @@ func testClientGatewayContainerSecurityMode(t *testing.T, sb integration.Sandbox
 		AllowedEntitlements: allowedEntitlements,
 	}
 	_, err = c.Build(ctx, solveOpts, product, b, nil)
-	require.NoError(t, err)
+
+	if expectFail {
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "security.insecure is not allowed")
+	} else {
+		require.NoError(t, err)
+	}
 
 	checkAllReleasable(t, c, sb, true)
 }
@@ -1805,7 +1833,15 @@ func testClientGatewayContainerExtraHosts(t *testing.T, sb integration.Sandbox)
 	checkAllReleasable(t, c, sb, true)
 }
 
-func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandbox) {
+func testClientGatewayContainerHostNetworkingAccess(t *testing.T, sb integration.Sandbox) {
+	testClientGatewayContainerHostNetworking(t, sb, false)
+}
+
+func testClientGatewayContainerHostNetworkingValidation(t *testing.T, sb integration.Sandbox) {
+	testClientGatewayContainerHostNetworking(t, sb, true)
+}
+
+func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandbox, expectFail bool) {
 	if os.Getenv("BUILDKIT_RUN_NETWORK_INTEGRATION_TESTS") == "" {
 		t.SkipNow()
 	}
@@ -1826,6 +1862,9 @@ func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandb
 	if sb.Value("netmode") == hostNetwork {
 		netMode = pb.NetMode_HOST
 		allowedEntitlements = []entitlements.Entitlement{entitlements.EntitlementNetworkHost}
+		if expectFail {
+			allowedEntitlements = []entitlements.Entitlement{}
+		}
 	}
 	c, err := New(sb.Context(), sb.Address())
 	require.NoError(t, err)
@@ -1884,7 +1923,12 @@ func testClientGatewayContainerHostNetworking(t *testing.T, sb integration.Sandb
 		t.Logf("Stderr: %q", stderr.String())
 
 		if netMode == pb.NetMode_HOST {
-			require.NoError(t, err)
+			if expectFail {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), "network.host is not allowed")
+			} else {
+				require.NoError(t, err)
+			}
 		} else {
 			require.Error(t, err)
 		}

client/client_test.go

@@ -162,6 +162,8 @@ func TestIntegration(t *testing.T) {
 		testUncompressedRegistryCacheImportExport,
 		testStargzLazyRegistryCacheImportExport,
 		testValidateDigestOrigin,
+		testMountStubsTimestamp,
+		testMountStubsDirectory,
 	)
 	tests = append(tests, diffOpTestCases()...)
 	integration.Run(t, tests, mirrors)
@@ -5888,3 +5890,118 @@ func fixedWriteCloser(wc io.WriteCloser) func(map[string]string) (io.WriteCloser
 		return wc, nil
 	}
 }
+
+func testMountStubsDirectory(t *testing.T, sb integration.Sandbox) {
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Image("busybox:latest").Run(
+		llb.Args([]string{"/bin/echo", "dummy"}),
+		llb.AddMount("/foo/bar", llb.Scratch(), llb.Tmpfs()),
+	)
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+	tarFile := filepath.Join(tmpDir, "out.tar")
+	tarFileW, err := os.Create(tarFile)
+	require.NoError(t, err)
+	defer tarFileW.Close()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(tarFileW),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+	tarFileW.Close()
+
+	tarFileR, err := os.Open(tarFile)
+	require.NoError(t, err)
+	defer tarFileR.Close()
+	tarR := tar.NewReader(tarFileR)
+
+	for {
+		hd, err := tarR.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+		if hd.Name == "foo/bar/" {
+			require.Fail(t, "foo/bar/ should not be in the tar")
+		}
+	}
+}
+
+// https://github.com/moby/buildkit/issues/3148
+func testMountStubsTimestamp(t *testing.T, sb integration.Sandbox) {
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	const sourceDateEpoch = int64(1234567890) // Fri Feb 13 11:31:30 PM UTC 2009
+	st := llb.Image("busybox:latest").Run(
+		llb.Args([]string{"/bin/touch", fmt.Sprintf("--date=@%d", sourceDateEpoch),
+			"/bin",
+			"/etc",
+			"/var",
+			"/var/foo",
+			"/tmp",
+			"/tmp/foo2",
+			"/tmp/foo2/bar",
+		}),
+		llb.AddMount("/var/foo", llb.Scratch(), llb.Tmpfs()),
+		llb.AddMount("/tmp/foo2/bar", llb.Scratch(), llb.Tmpfs()),
+	)
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	tmpDir := t.TempDir()
+	tarFile := filepath.Join(tmpDir, "out.tar")
+	tarFileW, err := os.Create(tarFile)
+	require.NoError(t, err)
+	defer tarFileW.Close()
+
+	_, err = c.Solve(sb.Context(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:   ExporterTar,
+				Output: fixedWriteCloser(tarFileW),
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+	tarFileW.Close()
+
+	tarFileR, err := os.Open(tarFile)
+	require.NoError(t, err)
+	defer tarFileR.Close()
+	tarR := tar.NewReader(tarFileR)
+	touched := map[string]*tar.Header{
+		"bin/": nil, // Regular dir
+		"etc/": nil, // Parent of file mounts (etc/{resolv.conf, hosts})
+		"var/": nil, // Parent of dir mount (var/foo/)
+		"tmp/": nil, // Grandparent of dir mount (tmp/foo2/bar/)
+		// No support for reproducing the timestamps of mount point directories such as var/foo/ and tmp/foo2/bar/,
+		// because the touched timestamp value is lost when the mount is unmounted.
+	}
+	for {
+		hd, err := tarR.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		require.NoError(t, err)
+		if x, ok := touched[hd.Name]; ok && x == nil {
+			touched[hd.Name] = hd
+		}
+	}
+	for name, hd := range touched {
+		t.Logf("Verifying %q (%+v)", name, hd)
+		require.NotNil(t, hd, name)
+		require.Equal(t, sourceDateEpoch, hd.ModTime.Unix(), name)
+	}
+}

cmd/buildkitd/main.go

@@ -636,8 +636,8 @@ func newController(c *cli.Context, cfg *config.Config) (*control.Controller, err
 		return nil, err
 	}
 	frontends := map[string]frontend.Frontend{}
-	frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc, dockerfile.Build)
-	frontends["gateway.v0"] = gateway.NewGatewayFrontend(wc)
+	frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build)
+	frontends["gateway.v0"] = gateway.NewGatewayFrontend(wc.Infos())
 
 	cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(cfg.Root, "cache.db"))
 	if err != nil {

executor/executor.go

@@ -6,7 +6,8 @@ import (
 	"net"
 	"syscall"
 
-	"github.com/moby/buildkit/snapshot"
+	"github.com/containerd/containerd/mount"
+	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/solver/pb"
 )
 
@@ -25,8 +26,13 @@ type Meta struct {
 	SecurityMode   pb.SecurityMode
 }
 
+type MountableRef interface {
+	Mount() ([]mount.Mount, func() error, error)
+	IdentityMapping() *idtools.IdentityMapping
+}
+
 type Mountable interface {
-	Mount(ctx context.Context, readonly bool) (snapshot.Mountable, error)
+	Mount(ctx context.Context, readonly bool) (MountableRef, error)
 }
 
 type Mount struct {

executor/oci/spec.go

@@ -11,7 +11,6 @@ import (
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/continuity/fs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/mitchellh/hashstructure/v2"
 	"github.com/moby/buildkit/executor"
@@ -198,6 +197,7 @@ func GenerateSpec(ctx context.Context, meta executor.Meta, mounts []executor.Mou
 type mountRef struct {
 	mount   mount.Mount
 	unmount func() error
+	subRefs map[string]mountRef
 }
 
 type submounts struct {
@@ -213,12 +213,19 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
 	}
 	h, err := hashstructure.Hash(m, hashstructure.FormatV2, nil)
 	if err != nil {
-		return mount.Mount{}, nil
+		return mount.Mount{}, err
 	}
 	if mr, ok := s.m[h]; ok {
-		sm, err := sub(mr.mount, subPath)
+		if sm, ok := mr.subRefs[subPath]; ok {
+			return sm.mount, nil
+		}
+		sm, unmount, err := sub(mr.mount, subPath)
 		if err != nil {
-			return mount.Mount{}, nil
+			return mount.Mount{}, err
+		}
+		mr.subRefs[subPath] = mountRef{
+			mount:   sm,
+			unmount: unmount,
 		}
 		return sm, nil
 	}
@@ -244,12 +251,17 @@ func (s *submounts) subMount(m mount.Mount, subPath string) (mount.Mount, error)
 			Options: opts,
 		},
 		unmount: lm.Unmount,
+		subRefs: map[string]mountRef{},
 	}
 
-	sm, err := sub(s.m[h].mount, subPath)
+	sm, unmount, err := sub(s.m[h].mount, subPath)
 	if err != nil {
 		return mount.Mount{}, err
 	}
+	s.m[h].subRefs[subPath] = mountRef{
+		mount:   sm,
+		unmount: unmount,
+	}
 	return sm, nil
 }
 
@@ -259,6 +271,9 @@ func (s *submounts) cleanup() {
 	for _, m := range s.m {
 		func(m mountRef) {
 			go func() {
+				for _, sm := range m.subRefs {
+					sm.unmount()
+				}
 				m.unmount()
 				wg.Done()
 			}()
@@ -267,15 +282,6 @@ func (s *submounts) cleanup() {
 	wg.Wait()
 }
 
-func sub(m mount.Mount, subPath string) (mount.Mount, error) {
-	src, err := fs.RootPath(m.Source, subPath)
-	if err != nil {
-		return mount.Mount{}, err
-	}
-	m.Source = src
-	return m, nil
-}
-
 func specMapping(s []idtools.IDMap) []specs.LinuxIDMapping {
 	var ids []specs.LinuxIDMapping
 	for _, item := range s {