bugfix: fix endpoints disappear when pouchd restart

Fix endpoints disappear when pouchd restart, restore network controller need keep endpoints that is using by the running containers, add the active sandboxes option to initialize network controller. Signed-off-by: Rudy Zhang <[email protected]>
AliyunContainerService · May 15, 2018 · a805494 · a805494
1 parent de46ffa
commit a805494
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 40 deletions.
diff --git a/daemon/daemon.go b/daemon/daemon.go
@@ -165,17 +165,18 @@ func (d *Daemon) Run() error {
 	}
 	d.volumeMgr = volumeMgr
 
-	networkMgr, err := internal.GenNetworkMgr(d.config, d)
+	containerMgr, err := internal.GenContainerMgr(ctx, d)
 	if err != nil {
 		return err
 	}
-	d.networkMgr = networkMgr
+	d.containerMgr = containerMgr
 
-	containerMgr, err := internal.GenContainerMgr(ctx, d)
+	networkMgr, err := internal.GenNetworkMgr(d.config, d)
 	if err != nil {
 		return err
 	}
-	d.containerMgr = containerMgr
+	d.networkMgr = networkMgr
+	containerMgr.(*mgr.ContainerManager).NetworkMgr = networkMgr
 
 	if err := d.addSystemLabels(); err != nil {
 		return err

diff --git a/daemon/mgr/container.go b/daemon/mgr/container.go
@@ -157,14 +157,13 @@ type ContainerManager struct {
 }
 
 // NewContainerManager creates a brand new container manager.
-func NewContainerManager(ctx context.Context, store *meta.Store, cli ctrd.APIClient, imgMgr ImageMgr, volMgr VolumeMgr, netMgr NetworkMgr, cfg *config.Config, contPlugin plugins.ContainerPlugin) (*ContainerManager, error) {
+func NewContainerManager(ctx context.Context, store *meta.Store, cli ctrd.APIClient, imgMgr ImageMgr, volMgr VolumeMgr, cfg *config.Config, contPlugin plugins.ContainerPlugin) (*ContainerManager, error) {
 	mgr := &ContainerManager{
 		Store:           store,
 		NameToID:        collect.NewSafeMap(),
 		Client:          cli,
 		ImageMgr:        imgMgr,
 		VolumeMgr:       volMgr,
-		NetworkMgr:      netMgr,
 		IOs:             containerio.NewCache(),
 		ExecProcesses:   collect.NewSafeMap(),
 		cache:           collect.NewSafeMap(),
@@ -2133,25 +2132,7 @@ func (mgr *ContainerManager) detachVolumes(ctx context.Context, c *Container, re
 }
 
 func (mgr *ContainerManager) buildContainerEndpoint(c *Container) *networktypes.Endpoint {
-	ep := &networktypes.Endpoint{
-		Owner:           c.ID,
-		Hostname:        c.Config.Hostname,
-		Domainname:      c.Config.Domainname,
-		HostsPath:       c.HostsPath,
-		ExtraHosts:      c.HostConfig.ExtraHosts,
-		HostnamePath:    c.HostnamePath,
-		ResolvConfPath:  c.ResolvConfPath,
-		NetworkDisabled: c.Config.NetworkDisabled,
-		NetworkMode:     c.HostConfig.NetworkMode,
-		DNS:             c.HostConfig.DNS,
-		DNSOptions:      c.HostConfig.DNSOptions,
-		DNSSearch:       c.HostConfig.DNSSearch,
-		MacAddress:      c.Config.MacAddress,
-		PublishAllPorts: c.HostConfig.PublishAllPorts,
-		ExposedPorts:    c.Config.ExposedPorts,
-		PortBindings:    c.HostConfig.PortBindings,
-		NetworkConfig:   c.NetworkSettings,
-	}
+	ep := BuildContainerEndpoint(c)
 
 	if mgr.containerPlugin != nil {
 		ep.Priority, ep.DisableResolver, ep.GenericParams = mgr.containerPlugin.PreCreateEndpoint(c.ID, c.Config.Env)

diff --git a/daemon/mgr/container_utils.go b/daemon/mgr/container_utils.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/alibaba/pouch/apis/types"
+	networktypes "github.com/alibaba/pouch/network/types"
 	"github.com/alibaba/pouch/pkg/errtypes"
 	"github.com/alibaba/pouch/pkg/meta"
 	"github.com/alibaba/pouch/pkg/randomid"
@@ -103,6 +104,29 @@ func (mgr *ContainerManager) generateName(id string) string {
 	return name
 }
 
+// BuildContainerEndpoint is used to build container's endpoint config.
+func BuildContainerEndpoint(c *Container) *networktypes.Endpoint {
+	return &networktypes.Endpoint{
+		Owner:           c.ID,
+		Hostname:        c.Config.Hostname,
+		Domainname:      c.Config.Domainname,
+		HostsPath:       c.HostsPath,
+		ExtraHosts:      c.HostConfig.ExtraHosts,
+		HostnamePath:    c.HostnamePath,
+		ResolvConfPath:  c.ResolvConfPath,
+		NetworkDisabled: c.Config.NetworkDisabled,
+		NetworkMode:     c.HostConfig.NetworkMode,
+		DNS:             c.HostConfig.DNS,
+		DNSOptions:      c.HostConfig.DNSOptions,
+		DNSSearch:       c.HostConfig.DNSSearch,
+		MacAddress:      c.Config.MacAddress,
+		PublishAllPorts: c.HostConfig.PublishAllPorts,
+		ExposedPorts:    c.Config.ExposedPorts,
+		PortBindings:    c.HostConfig.PortBindings,
+		NetworkConfig:   c.NetworkSettings,
+	}
+}
+
 func parseSecurityOpts(c *Container, securityOpts []string) error {
 	var (
 		labelOpts []string

diff --git a/daemon/mgr/network.go b/daemon/mgr/network.go
@@ -65,13 +65,32 @@ type NetworkManager struct {
 }
 
 // NewNetworkManager creates a brand new network manager.
-func NewNetworkManager(cfg *config.Config, store *meta.Store) (*NetworkManager, error) {
+func NewNetworkManager(cfg *config.Config, store *meta.Store, ctrMgr ContainerMgr) (*NetworkManager, error) {
 	// Create a new controller instance
 	cfg.NetworkConfg.MetaPath = path.Dir(store.BaseDir)
 	cfg.NetworkConfg.ExecRoot = network.DefaultExecRoot
 
 	initNetworkLog(cfg)
 
+	// get active sandboxes
+	ctrs, err := ctrMgr.List(context.Background(),
+		func(c *Container) bool {
+			return (c.IsRunning() || c.IsPaused()) && !isContainer(c.HostConfig.NetworkMode)
+		}, &ContainerListOption{All: true})
+	if err != nil {
+		logrus.Errorf("failed to new network manager, can not get container list")
+		return nil, errors.Wrap(err, "failed to get container list")
+	}
+	cfg.NetworkConfg.ActiveSandboxes = make(map[string]interface{})
+	for _, c := range ctrs {
+		endpoint := BuildContainerEndpoint(c)
+		sbOptions, err := buildSandboxOptions(cfg.NetworkConfg, endpoint)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to build sandbox options")
+		}
+		cfg.NetworkConfg.ActiveSandboxes[c.NetworkSettings.SandboxID] = sbOptions
+	}
+
 	ctlOptions, err := controllerOptions(cfg.NetworkConfg)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to build network options")
@@ -280,7 +299,7 @@ func (nm *NetworkManager) EndpointCreate(ctx context.Context, endpoint *types.En
 	// create sandbox
 	sb := nm.getNetworkSandbox(containerID)
 	if sb == nil {
-		sandboxOptions, err := nm.sandboxOptions(endpoint)
+		sandboxOptions, err := buildSandboxOptions(nm.config, endpoint)
 		if err != nil {
 			return "", fmt.Errorf("failed to build sandbox options(%v)", err)
 		}
@@ -422,6 +441,10 @@ func controllerOptions(cfg network.Config) ([]nwconfig.Option, error) {
 		options = append(options, nwconfig.OptionExecRoot(cfg.ExecRoot))
 	}
 
+	if len(cfg.ActiveSandboxes) != 0 {
+		options = append(options, nwconfig.OptionActiveSandboxes(cfg.ActiveSandboxes))
+	}
+
 	options = append(options, nwconfig.OptionDefaultDriver("bridge"))
 	options = append(options, nwconfig.OptionDefaultNetwork("bridge"))
 
@@ -562,7 +585,7 @@ func endpointOptions(n libnetwork.Network, endpoint *types.Endpoint) ([]libnetwo
 	return createOptions, nil
 }
 
-func (nm *NetworkManager) sandboxOptions(endpoint *types.Endpoint) ([]libnetwork.SandboxOption, error) {
+func buildSandboxOptions(config network.Config, endpoint *types.Endpoint) ([]libnetwork.SandboxOption, error) {
 	var (
 		sandboxOptions []libnetwork.SandboxOption
 		dns            []string
@@ -577,9 +600,9 @@ func (nm *NetworkManager) sandboxOptions(endpoint *types.Endpoint) ([]libnetwork
 		if len(endpoint.ExtraHosts) == 0 {
 			sandboxOptions = append(sandboxOptions, libnetwork.OptionOriginHostsPath("/etc/hosts"))
 		}
-		if len(endpoint.DNS) == 0 && len(nm.config.DNS) == 0 &&
-			len(endpoint.DNSSearch) == 0 && len(nm.config.DNSSearch) == 0 &&
-			len(endpoint.DNSOptions) == 0 && len(nm.config.DNSOptions) == 0 {
+		if len(endpoint.DNS) == 0 && len(config.DNS) == 0 &&
+			len(endpoint.DNSSearch) == 0 && len(config.DNSSearch) == 0 &&
+			len(endpoint.DNSOptions) == 0 && len(config.DNSOptions) == 0 {
 			sandboxOptions = append(sandboxOptions, libnetwork.OptionOriginResolvConfPath("/etc/resolv.conf"))
 		}
 	} else {
@@ -592,8 +615,8 @@ func (nm *NetworkManager) sandboxOptions(endpoint *types.Endpoint) ([]libnetwork
 	// parse DNS
 	if len(endpoint.DNS) > 0 {
 		dns = endpoint.DNS
-	} else if len(nm.config.DNS) > 0 {
-		dns = nm.config.DNS
+	} else if len(config.DNS) > 0 {
+		dns = config.DNS
 	}
 	for _, d := range dns {
 		sandboxOptions = append(sandboxOptions, libnetwork.OptionDNS(d))
@@ -602,8 +625,8 @@ func (nm *NetworkManager) sandboxOptions(endpoint *types.Endpoint) ([]libnetwork
 	// parse DNS Search
 	if len(endpoint.DNSSearch) > 0 {
 		dnsSearch = endpoint.DNSSearch
-	} else if len(nm.config.DNSSearch) > 0 {
-		dnsSearch = nm.config.DNSSearch
+	} else if len(config.DNSSearch) > 0 {
+		dnsSearch = config.DNSSearch
 	}
 	for _, ds := range dnsSearch {
 		sandboxOptions = append(sandboxOptions, libnetwork.OptionDNSSearch(ds))
@@ -612,8 +635,8 @@ func (nm *NetworkManager) sandboxOptions(endpoint *types.Endpoint) ([]libnetwork
 	// parse DNS Options
 	if len(endpoint.DNSOptions) > 0 {
 		dnsOptions = endpoint.DNSOptions
-	} else if len(nm.config.DNSOptions) > 0 {
-		dnsOptions = nm.config.DNSOptions
+	} else if len(config.DNSOptions) > 0 {
+		dnsOptions = config.DNSOptions
 	}
 	for _, ds := range dnsOptions {
 		sandboxOptions = append(sandboxOptions, libnetwork.OptionDNSOptions(ds))

diff --git a/internal/generator.go b/internal/generator.go
@@ -25,7 +25,7 @@ type DaemonProvider interface {
 
 // GenContainerMgr generates a ContainerMgr instance according to config cfg.
 func GenContainerMgr(ctx context.Context, d DaemonProvider) (mgr.ContainerMgr, error) {
-	return mgr.NewContainerManager(ctx, d.MetaStore(), d.Containerd(), d.ImgMgr(), d.VolMgr(), d.NetMgr(), d.Config(), d.ContainerPlugin())
+	return mgr.NewContainerManager(ctx, d.MetaStore(), d.Containerd(), d.ImgMgr(), d.VolMgr(), d.Config(), d.ContainerPlugin())
 }
 
 // GenSystemMgr generates a SystemMgr instance according to config cfg.
@@ -47,7 +47,7 @@ func GenVolumeMgr(cfg *config.Config, d DaemonProvider) (mgr.VolumeMgr, error) {
 
 // GenNetworkMgr generates a NetworkMgr instance according to config cfg.
 func GenNetworkMgr(cfg *config.Config, d DaemonProvider) (mgr.NetworkMgr, error) {
-	return mgr.NewNetworkManager(cfg, d.MetaStore())
+	return mgr.NewNetworkManager(cfg, d.MetaStore(), d.CtrMgr())
 }
 
 // GenCriMgr generates a CriMgr instance.

diff --git a/network/config.go b/network/config.go
@@ -15,6 +15,8 @@ type Config struct {
 
 	// bridge config
 	BridgeConfig BridgeConfig
+
+	ActiveSandboxes map[string]interface{}
 }
 
 // BridgeConfig defines the bridge network configuration.

diff --git a/network/mode/init.go b/network/mode/init.go
@@ -12,6 +12,12 @@ import (
 
 // NetworkModeInit is used to initilize network mode, include host and none network.
 func NetworkModeInit(ctx context.Context, config network.Config, manager mgr.NetworkMgr) error {
+	// if it has old containers, don't to intialize network.
+	if len(config.ActiveSandboxes) > 0 {
+		logrus.Warnf("There are old containers, don't to initialize network")
+		return nil
+	}
+
 	// init none network
 	if n, _ := manager.Get(ctx, "none"); n == nil {
 		logrus.Debugf("create none network")

diff --git a/test/z_cli_daemon_test.go b/test/z_cli_daemon_test.go
@@ -288,7 +288,7 @@ func (suite *PouchDaemonSuite) TestDaemonRestartWithPausedContainer(c *check.C)
 	cname := "TestDaemonRestartWithPausedContainer"
 	{
 		result := RunWithSpecifiedDaemon(dcfg, "run", "-d", "--name", cname,
-			"-p", "1234:80", busyboxImage, "top")
+			"-p", "5678:80", busyboxImage, "top")
 		if result.ExitCode != 0 {
 			dcfg.DumpLog()
 			c.Fatalf("run container failed, err: %v", result)