Add (hidden) flags to set containerd namespaces

This allows our tests, which all share a containerd instance, to be a bit more isolated by setting the containerd namespaces to the generated daemon ID's rather than the default namespaces. This came about because I found in some cases we had test daemons failing to start (really very slow to start) because it was (seemingly) processing events from other tests. Signed-off-by: Brian Goff <[email protected]>
thaJeztah · Jul 12, 2019 · 24ad2f4 · 24ad2f4
1 parent 2fc3480
commit 24ad2f4
Show file tree

Hide file tree

Showing 7 changed files with 32 additions and 11 deletions.
diff --git a/cmd/dockerd/config.go b/cmd/dockerd/config.go
@@ -3,8 +3,10 @@ package main
 import (
 	"runtime"
 
+	"github.com/docker/docker/daemon"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/opts"
+	"github.com/docker/docker/plugin/executor/containerd"
 	"github.com/docker/docker/registry"
 	"github.com/spf13/pflag"
 )
@@ -85,7 +87,13 @@ func installCommonConfigFlags(conf *config.Config, flags *pflag.FlagSet) error {
 
 	conf.MaxConcurrentDownloads = &maxConcurrentDownloads
 	conf.MaxConcurrentUploads = &maxConcurrentUploads
-	return nil
+
+	flags.StringVar(&conf.ContainerdNamespace, "containerd-namespace", daemon.ContainersNamespace, "Containerd namespace to use")
+	if err := flags.MarkHidden("containerd-namespace"); err != nil {
+		return err
+	}
+	flags.StringVar(&conf.ContainerdPluginNamespace, "containerd-plugins-namespace", containerd.PluginNamespace, "Containerd namespace to use for plugins")
+	return flags.MarkHidden("containerd-plugins-namespace")
 }
 
 func installRegistryServiceFlags(options *registry.ServiceOptions, flags *pflag.FlagSet) {

diff --git a/daemon/config/config.go b/daemon/config/config.go
@@ -230,6 +230,9 @@ type CommonConfig struct {
 	Features map[string]bool `json:"features,omitempty"`
 
 	Builder BuilderConfig `json:"builder,omitempty"`
+
+	ContainerdNamespace       string `json:"containerd-namespace,omitempty"`
+	ContainerdPluginNamespace string `json:"containerd-plugin-namespace,omitempty"`
 }
 
 // IsValueSet returns true if a configuration value

diff --git a/daemon/daemon.go b/daemon/daemon.go
@@ -888,7 +888,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
 	}
 	if config.ContainerdAddr != "" {
-		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(ContainersNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+		d.containerdCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 		}
@@ -900,13 +900,13 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		// Windows is not currently using containerd, keep the
 		// client as nil
 		if config.ContainerdAddr != "" {
-			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(pluginexec.PluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
+			pluginCli, err = containerd.New(config.ContainerdAddr, containerd.WithDefaultNamespace(config.ContainerdPluginNamespace), containerd.WithDialOpts(gopts), containerd.WithTimeout(60*time.Second))
 			if err != nil {
 				return nil, errors.Wrapf(err, "failed to dial %q", config.ContainerdAddr)
 			}
 		}
 
-		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, m)
+		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m)
 	}
 
 	// Plugin system initialization should happen before restore. Do not change order.
@@ -1054,7 +1054,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 
 	go d.execCommandGC()
 
-	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), ContainersNamespace, d)
+	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
 	if err != nil {
 		return nil, err
 	}

diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go
@@ -757,6 +757,9 @@ func (daemon *Daemon) initRuntimes(runtimes map[string]types.Runtime) (err error
 
 // verifyDaemonSettings performs validation of daemon config struct
 func verifyDaemonSettings(conf *config.Config) error {
+	if conf.ContainerdNamespace == conf.ContainerdPluginNamespace {
+		return errors.New("containers namespace and plugins namespace cannot be the same")
+	}
 	// Check for mutually incompatible config options
 	if conf.BridgeConfig.Iface != "" && conf.BridgeConfig.IP != "" {
 		return fmt.Errorf("You specified -b & --bip, mutually exclusive options. Please specify only one")

diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go
@@ -25,7 +25,6 @@ import (
 
 	"github.com/cloudflare/cfssl/helpers"
 	"github.com/docker/docker/api/types"
-	moby_daemon "github.com/docker/docker/daemon"
 	"github.com/docker/docker/integration-cli/checker"
 	"github.com/docker/docker/integration-cli/cli"
 	"github.com/docker/docker/integration-cli/cli/build"
@@ -1457,7 +1456,7 @@ func (s *DockerDaemonSuite) TestCleanupMountsAfterDaemonAndContainerKill(c *chec
 
 	// kill the container
 	icmd.RunCommand(ctrBinary, "--address", containerdSocket,
-		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", id).Assert(c, icmd.Success)
+		"--namespace", d.ContainersNamespace(), "tasks", "kill", id).Assert(c, icmd.Success)
 
 	// restart daemon.
 	d.Restart(c)
@@ -1977,7 +1976,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithKilledRunningContainer(t *check
 
 	// kill the container
 	icmd.RunCommand(ctrBinary, "--address", containerdSocket,
-		"--namespace", moby_daemon.ContainersNamespace, "tasks", "kill", cid).Assert(t, icmd.Success)
+		"--namespace", s.d.ContainersNamespace(), "tasks", "kill", cid).Assert(t, icmd.Success)
 
 	// Give time to containerd to process the command if we don't
 	// the exit event might be received after we do the inspect
@@ -2080,7 +2079,7 @@ func (s *DockerDaemonSuite) TestDaemonRestartWithUnpausedRunningContainer(t *che
 	result := icmd.RunCommand(
 		ctrBinary,
 		"--address", containerdSocket,
-		"--namespace", moby_daemon.ContainersNamespace,
+		"--namespace", s.d.ContainersNamespace(),
 		"tasks", "resume", cid)
 	result.Assert(t, icmd.Success)
 

diff --git a/internal/test/daemon/daemon.go b/internal/test/daemon/daemon.go
@@ -137,6 +137,11 @@ func New(t testingT, ops ...func(*Daemon)) *Daemon {
 	return d
 }
 
+// ContainersNamespace returns the containerd namespace used for containers.
+func (d *Daemon) ContainersNamespace() string {
+	return d.id
+}
+
 // RootDir returns the root directory of the daemon.
 func (d *Daemon) RootDir() string {
 	return d.Root
@@ -226,12 +231,15 @@ func (d *Daemon) StartWithLogFile(out *os.File, providedArgs ...string) error {
 	if err != nil {
 		return errors.Wrapf(err, "[%s] could not find docker binary in $PATH", d.id)
 	}
+
 	args := append(d.GlobalFlags,
 		"--containerd", containerdSocket,
 		"--data-root", d.Root,
 		"--exec-root", d.execRoot,
 		"--pidfile", fmt.Sprintf("%s/docker.pid", d.Folder),
 		fmt.Sprintf("--userland-proxy=%t", d.userlandProxy),
+		"--containerd-namespace", d.id,
+		"--containerd-plugins-namespace", d.id+"p",
 	)
 	if d.defaultCgroupNamespaceMode != "" {
 		args = append(args, []string{"--default-cgroupns-mode", d.defaultCgroupNamespaceMode}...)

diff --git a/plugin/executor/containerd/containerd.go b/plugin/executor/containerd/containerd.go
@@ -26,13 +26,13 @@ type ExitHandler interface {
 }
 
 // New creates a new containerd plugin executor
-func New(ctx context.Context, rootDir string, cli *containerd.Client, exitHandler ExitHandler) (*Executor, error) {
+func New(ctx context.Context, rootDir string, cli *containerd.Client, ns string, exitHandler ExitHandler) (*Executor, error) {
 	e := &Executor{
 		rootDir:     rootDir,
 		exitHandler: exitHandler,
 	}
 
-	client, err := libcontainerd.NewClient(ctx, cli, rootDir, PluginNamespace, e)
+	client, err := libcontainerd.NewClient(ctx, cli, rootDir, ns, e)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating containerd exec client")
 	}