diff --git a/api/api_test.go b/api/api_test.go index e78e7c108e..02e266d910 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -87,7 +87,8 @@ func createTestNetwork(t *testing.T, network string) (libnetwork.NetworkControll // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -192,7 +193,8 @@ func TestCreateDeleteNetwork(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -267,7 +269,8 @@ func TestGetNetworksAndEndpoints(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -526,7 +529,8 @@ func TestProcGetServices(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -999,7 +1003,8 @@ func TestDetectGetNetworksInvalidQueryComposition(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -1114,7 +1119,8 @@ func TestCreateDeleteEndpoints(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -1240,7 +1246,8 @@ func TestJoinLeave(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -1691,7 +1698,8 @@ func TestHttpHandlerUninit(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -1761,7 +1769,8 @@ func TestHttpHandlerBadBody(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -1797,7 +1806,8 @@ func TestEndToEnd(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } @@ -2249,7 +2259,8 @@ func TestEndToEndErrorMessage(t *testing.T) { // Cleanup local datastore file os.Remove(datastore.DefaultScopes("")[datastore.LocalScope].Client.Address) - c, err := libnetwork.New() + old := make(map[string]interface{}) + c, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } diff --git a/cmd/dnet/dnet.go b/cmd/dnet/dnet.go index 885691f009..a986cb73cd 100644 --- a/cmd/dnet/dnet.go +++ b/cmd/dnet/dnet.go @@ -243,7 +243,8 @@ func (d *dnetConnection) dnetDaemon(cfgFile string) error { cOptions = append(cOptions, config.OptionDriverConfig("bridge", bridgeOption)) - controller, err := libnetwork.New(cOptions...) + old := make(map[string]interface{}) + controller, _, err := libnetwork.New(old, cOptions...) if err != nil { fmt.Println("Error starting dnetDaemon :", err) return err diff --git a/cmd/ovrouter/ovrouter.go b/cmd/ovrouter/ovrouter.go index 8e286d0729..54df0d0e1f 100644 --- a/cmd/ovrouter/ovrouter.go +++ b/cmd/ovrouter/ovrouter.go @@ -79,6 +79,10 @@ func (ep *endpoint) SetNames(srcName, dstPrefix string) error { return nil } +func (ep *endpoint) SrcName() string { + return ep.name +} + func (ep *endpoint) SetGateway(net.IP) error { return nil } diff --git a/cmd/readme_test/readme.go b/cmd/readme_test/readme.go index 600f99bcff..9874a6ba7b 100644 --- a/cmd/readme_test/readme.go +++ b/cmd/readme_test/readme.go @@ -23,7 +23,8 @@ func main() { driverOptions := options.Generic{} genericOption := make(map[string]interface{}) genericOption[netlabel.GenericData] = driverOptions - controller, err := libnetwork.New(config.OptionDriverConfig(networkType, genericOption)) + old := make(map[string]interface{}) + controller, _, err := libnetwork.New(old, config.OptionDriverConfig(networkType, genericOption)) if err != nil { log.Fatalf("libnetwork.New: %s", err) } diff --git a/controller.go b/controller.go index fa14b1cf1f..56d9378936 100644 --- a/controller.go +++ b/controller.go @@ -147,7 +147,7 @@ type initializer struct { } // New creates a new instance of network controller. -func New(cfgOptions ...config.Option) (NetworkController, error) { +func New(oldRunningContainers map[string]interface{}, cfgOptions ...config.Option) (NetworkController, map[string]interface{}, error) { c := &controller{ id: stringid.GenerateRandomID(), cfg: config.ParseConfigOptions(cfgOptions...), @@ -157,20 +157,20 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } if err := c.agentInit(c.cfg.Daemon.Bind); err != nil { - return nil, err + return nil, nil, err } if err := c.agentJoin(c.cfg.Daemon.Neighbors); err != nil { - return nil, err + return nil, nil, err } if err := c.initStores(); err != nil { - return nil, err + return nil, nil, err } drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil) if err != nil { - return nil, err + return nil, nil, err } for _, i := range getInitializers() { @@ -183,7 +183,7 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil { - return nil, err + return nil, nil, err } } c.drvRegistry = drvRegistry @@ -196,15 +196,79 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } } - c.sandboxCleanup() + c.sandboxCleanup(oldRunningContainers) + if err := c.restoreSandbox(oldRunningContainers); err != nil { + log.Errorf("failed to restore sandbox") + } + c.cleanupLocalEndpoints() c.networkCleanup() if err := c.startExternalKeyListener(); err != nil { - return nil, err + return nil, nil, err + } + restored := make(map[string]interface{}) + for _, sb := range c.sandboxes { + restored[sb.ContainerID()] = true } - return c, nil + return c, restored, nil +} + +func (c *controller) restoreSandbox(sbids map[string]interface{}) error { + for id, sb := range c.sandboxes { + log.Infof("restore sandbox %s of container %s", sb.ID(), sb.ContainerID()) + option, ok := sbids[sb.ContainerID()].([]SandboxOption) + if !ok { + log.Errorf("failed to restore sandbox: no restore options passed from daemon") + delete(c.sandboxes, id) + continue + } + err := sb.restoreSandbox(option) + if err != nil { + log.Errorf("failed to restore sandbox %s", sb.ID()) + delete(c.sandboxes, id) + continue + } + // restore endpoints in this sandbox + // Fixme: if one of the endpoints failed to restore, should we delete this sandbox? + for _, ep := range sb.endpoints { + log.Infof("restore endpoint %s", ep.ID()) + c.watchSvcRecord(ep) + net, err := ep.getNetworkFromStore() + if err != nil { + log.Errorf("Restore sandbox: failed to get endpoint network from store: %v", err) + ep.Delete(true) + continue + } + d, err := net.driver(true) + if err != nil { + log.Errorf("Resore sandbox: failed to get driver of endpoint %s: %v", ep.ID(), err) + ep.Delete(true) + continue + } + options := make(map[string]interface{}) + for key, value := range options { + options[key] = value + } + for key, value := range sb.Labels() { + options[key] = value + } + + err = d.Restore(net.ID(), ep.id, sb.Key(), ep.Interface(), options) + if err != nil { + log.Errorf("Restore sandbox: failed to restore endpoint %s to driver of network %s: %v", ep.Name(), net.Name(), err) + ep.Delete(true) + continue + } + } + if sb.config.useDefaultSandBox { + c.sboxOnce.Do(func() { + c.defOsSbox = sb.osSbox + }) + } + } + return nil } func (c *controller) makeDriverConfig(ntype string) map[string]interface{} { diff --git a/driverapi/driverapi.go b/driverapi/driverapi.go index f555246ceb..72217748ce 100644 --- a/driverapi/driverapi.go +++ b/driverapi/driverapi.go @@ -57,6 +57,9 @@ type Driver interface { // Leave method is invoked when a Sandbox detaches from an endpoint. Leave(nid, eid string) error + // Restore reconstruct driver struct + Restore(nid, eid string, sboxKey string, ifInfo InterfaceInfo, options map[string]interface{}) error + // ProgramExternalConnectivity invokes the driver method which does the necessary // programming to allow the external connectivity dictated by the passed options ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error @@ -103,6 +106,9 @@ type InterfaceInfo interface { // AddressIPv6 returns the IPv6 address. AddressIPv6() *net.IPNet + + // SrcName return the srcName + SrcName() string } // InterfaceNameInfo provides a go interface for the drivers to assign names diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index baa38db5a4..f79170b74f 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -376,7 +376,8 @@ func (d *driver) configure(option map[string]interface{}) error { logrus.Warnf("Running modprobe bridge br_netfilter failed with message: %s, error: %v", out, err) } } - removeIPChains() + // TODO: need a better way to handle this + // removeIPChains() natChain, filterChain, isolationChain, err = setupIPChains(config) if err != nil { return err @@ -1211,11 +1212,10 @@ func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string } // Program any required port mapping and store them in the endpoint - endpoint.portMapping, err = network.allocatePorts(endpoint, network.config.DefaultBindingIP, d.config.EnableUserlandProxy) + endpoint.portMapping, err = network.allocatePorts(endpoint, network.config.DefaultBindingIP, d.config.EnableUserlandProxy, false) if err != nil { return err } - if !network.config.EnableICC { return d.link(network, endpoint, true) } @@ -1340,6 +1340,82 @@ func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{ return nil } +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + // restore endpoint + if ifInfo == nil { + return errors.New("invalid interface info passed") + } + + // Get the network handler and make sure it exists + d.Lock() + n, ok := d.networks[nid] + d.Unlock() + + if !ok { + return types.NotFoundErrorf("network %s does not exist", nid) + } + if n == nil { + return driverapi.ErrNoNetwork(nid) + } + + // Sanity check + n.Lock() + if n.id != nid { + n.Unlock() + return InvalidNetworkIDError(nid) + } + n.Unlock() + + // Check if endpoint id is good and retrieve correspondent endpoint + ep, err := n.getEndpoint(eid) + if err != nil { + return err + } + + // Endpoint with that id exists either on desired or other sandbox + if ep != nil { + return driverapi.ErrEndpointExists(eid) + } + + // Try to convert the options to endpoint configuration + epConfig, err := parseEndpointOptions(options) + if err != nil { + return err + } + endpoint := &bridgeEndpoint{id: eid, config: epConfig} + endpoint.macAddress = ifInfo.MacAddress() + endpoint.addr = ifInfo.Address() + endpoint.addrv6 = ifInfo.AddressIPv6() + endpoint.srcName = ifInfo.SrcName() + + endpoint.containerConfig, err = parseContainerOptions(options) + if err != nil { + return err + } + endpoint.extConnConfig, err = parseConnectivityOptions(options) + if err != nil { + return err + } + + if len(endpoint.extConnConfig.PortBindings) > 0 { + endpoint.portMapping, err = n.allocatePorts(endpoint, n.config.DefaultBindingIP, d.config.EnableUserlandProxy, true) + if err != nil { + return err + } + + // This is to make sure that all the iptalbes rules are still exist + // If the rule not exist, re-create it + if !n.config.EnableICC { + return d.link(n, endpoint, true) + } + + } + n.Lock() + n.endpoints[eid] = endpoint + n.Unlock() + return nil +} + func parseEndpointOptions(epOptions map[string]interface{}) (*endpointConfiguration, error) { if epOptions == nil { return nil, nil diff --git a/drivers/bridge/bridge_store.go b/drivers/bridge/bridge_store.go index de9635289a..23ff349d29 100644 --- a/drivers/bridge/bridge_store.go +++ b/drivers/bridge/bridge_store.go @@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool { } func (ncfg *networkConfiguration) Skip() bool { - return ncfg.DefaultBridge + return false } func (ncfg *networkConfiguration) New() datastore.KVObject { diff --git a/drivers/bridge/bridge_test.go b/drivers/bridge/bridge_test.go index 988df41ac5..9b949414df 100644 --- a/drivers/bridge/bridge_test.go +++ b/drivers/bridge/bridge_test.go @@ -367,6 +367,10 @@ func (i *testInterface) Address() *net.IPNet { return i.addr } +func (i *testInterface) SrcName() string { + return i.srcName +} + func (i *testInterface) AddressIPv6() *net.IPNet { return i.addrv6 } diff --git a/drivers/bridge/port_mapping.go b/drivers/bridge/port_mapping.go index 965cc9a039..fa61aa3be3 100644 --- a/drivers/bridge/port_mapping.go +++ b/drivers/bridge/port_mapping.go @@ -14,7 +14,7 @@ var ( defaultBindingIP = net.IPv4(0, 0, 0, 0) ) -func (n *bridgeNetwork) allocatePorts(ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { +func (n *bridgeNetwork) allocatePorts(ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled, restore bool) ([]types.PortBinding, error) { if ep.extConnConfig == nil || ep.extConnConfig.PortBindings == nil { return nil, nil } @@ -24,14 +24,14 @@ func (n *bridgeNetwork) allocatePorts(ep *bridgeEndpoint, reqDefBindIP net.IP, u defHostIP = reqDefBindIP } - return n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled) + return n.allocatePortsInternal(ep.extConnConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled, restore) } -func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { +func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled, restore bool) ([]types.PortBinding, error) { bs := make([]types.PortBinding, 0, len(bindings)) for _, c := range bindings { b := c.GetCopy() - if err := n.allocatePort(&b, containerIP, defHostIP, ulPxyEnabled); err != nil { + if err := n.allocatePort(&b, containerIP, defHostIP, ulPxyEnabled, restore); err != nil { // On allocation failure, release previously allocated ports. On cleanup error, just log a warning message if cuErr := n.releasePortsInternal(bs); cuErr != nil { logrus.Warnf("Upon allocation failure for %v, failed to clear previously allocated port bindings: %v", b, cuErr) @@ -43,7 +43,7 @@ func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, cont return bs, nil } -func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) error { +func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled, restore bool) error { var ( host net.Addr err error @@ -70,7 +70,7 @@ func (n *bridgeNetwork) allocatePort(bnd *types.PortBinding, containerIP, defHos // Try up to maxAllocatePortAttempts times to get a port that's not already allocated. for i := 0; i < maxAllocatePortAttempts; i++ { - if host, err = n.portMapper.MapRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortEnd), ulPxyEnabled); err == nil { + if host, err = n.portMapper.MapRange(container, bnd.HostIP, int(bnd.HostPort), int(bnd.HostPortEnd), ulPxyEnabled, restore); err == nil { break } // There is no point in immediately retrying to map an explicitly chosen port. diff --git a/drivers/host/host.go b/drivers/host/host.go index bec64465a0..33857aa1d8 100644 --- a/drivers/host/host.go +++ b/drivers/host/host.go @@ -95,3 +95,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} diff --git a/drivers/ipvlan/ipvlan.go b/drivers/ipvlan/ipvlan.go index 8ea44fcbb4..7a42465337 100644 --- a/drivers/ipvlan/ipvlan.go +++ b/drivers/ipvlan/ipvlan.go @@ -1,6 +1,7 @@ package ipvlan import ( + "fmt" "net" "sync" @@ -101,3 +102,20 @@ func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) { } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + n, err := d.getNetwork(nid) + if err != nil { + return fmt.Errorf("network id %q not found", nid) + } + ep := &endpoint{ + id: eid, + addr: ifInfo.Address(), + addrv6: ifInfo.AddressIPv6(), + mac: ifInfo.MacAddress(), + srcName: ifInfo.SrcName(), + } + + n.addEndpoint(ep) + return nil +} diff --git a/drivers/macvlan/macvlan.go b/drivers/macvlan/macvlan.go index 5ace97f90c..965217292e 100644 --- a/drivers/macvlan/macvlan.go +++ b/drivers/macvlan/macvlan.go @@ -1,6 +1,7 @@ package macvlan import ( + "fmt" "net" "sync" @@ -103,3 +104,19 @@ func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{ func (d *driver) EventNotify(etype driverapi.EventType, nid, tableName, key string, value []byte) { } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + n, err := d.getNetwork(nid) + if err != nil { + return fmt.Errorf("network id %q not found", nid) + } + ep := &endpoint{ + id: eid, + addr: ifInfo.Address(), + addrv6: ifInfo.AddressIPv6(), + mac: ifInfo.MacAddress(), + srcName: ifInfo.SrcName(), + } + n.addEndpoint(ep) + return nil +} diff --git a/drivers/null/null.go b/drivers/null/null.go index a137b000fa..5ad531a74f 100644 --- a/drivers/null/null.go +++ b/drivers/null/null.go @@ -95,3 +95,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} diff --git a/drivers/overlay/ov_network.go b/drivers/overlay/ov_network.go index 893f8da314..28e4560f4d 100644 --- a/drivers/overlay/ov_network.go +++ b/drivers/overlay/ov_network.go @@ -176,6 +176,81 @@ func (n *network) incEndpointCount() { n.joinCnt++ } +func (n *network) restoreSandbox(s *subnet) error { + n.once.Do(func() { + // save the error status of initSandbox in n.initErr so that + // all the racing go routines are able to know the status. + n.initErr = n.restore(s) + }) + + return n.initErr +} + +func (n *network) restore(s *subnet) error { + hostModeOnce.Do(func() { + if os.Getenv("_OVERLAY_HOST_MODE") != "" { + hostMode = true + } + }) + key := osl.GenerateKey("-" + n.id) + if key == "" { + return fmt.Errorf("failed to find old sandbox key %s", osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id)) + } + sbox := osl.NewNullSandbox(key) + + n.setSandbox(sbox) + var reErr error + s.once.Do(func() { + brName := n.generateBridgeName(s) + vxlanName := n.generateVxlanName(s) + // restore osl sandbox + Ifaces := make(map[string][]osl.IfaceOption) + brIfaceOption := make([]osl.IfaceOption, 2) + brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP)) + brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true)) + Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption + // we should restore the bridge interface first + err := sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + reErr = err + return + } + + Ifaces = make(map[string][]osl.IfaceOption) + vxlanIfaceOption := make([]osl.IfaceOption, 1) + vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName)) + Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption + + err = sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + reErr = err + return + } + n.Lock() + s.vxlanName = vxlanName + s.brName = brName + n.Unlock() + + }) + + if reErr != nil { + return reErr + } + var ( + nlSock *nl.NetlinkSocket + err error + ) + sbox.InvokeFunc(func() { + nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH) + if err != nil { + err = fmt.Errorf("failed to subscribe to neighbor group netlink messages") + } + }) + + go n.watchMiss(nlSock) + return nil +} + func (n *network) joinSandbox() error { // If there is a race between two go routines here only one will win // the other will wait. diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index cfdebb5072..1498bd622d 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -13,6 +13,7 @@ import ( "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/idm" "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/osl" "github.com/docker/libnetwork/types" "github.com/hashicorp/serf/serf" ) @@ -238,3 +239,47 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + n := d.network(nid) + if n == nil { + return fmt.Errorf("network id %q not found", nid) + } + + ep := &endpoint{ + id: eid, + addr: ifInfo.Address(), + mac: ifInfo.MacAddress(), + ifName: ifInfo.SrcName(), + } + + n.addEndpoint(ep) + + s := n.getSubnetforIP(ep.addr) + if s == nil { + return fmt.Errorf("could not find subnet for endpoint %s", eid) + } + + if err := n.obtainVxlanID(s); err != nil { + return fmt.Errorf("couldn't get vxlan id for %q: %v", s.subnetIP.String(), err) + } + + if err := n.restoreSandbox(s); err != nil { + return fmt.Errorf("failed to restore overlay sandbox: %v", err) + } + + Ifaces := make(map[string][]osl.IfaceOption) + vethIfaceOption := make([]osl.IfaceOption, 1) + vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName)) + Ifaces[fmt.Sprintf("%s+%s", "veth", "veth")] = vethIfaceOption + + err := n.sbox.Restore(Ifaces, nil, nil, nil) + if err != nil { + return fmt.Errorf("failed to restore overlay sandbox: %v", err) + } + + n.incEndpointCount() + d.peerDbAdd(nid, eid, ep.addr.IP, ep.addr.Mask, ep.mac, + net.ParseIP(d.bindAddress), true) + return nil +} diff --git a/drivers/overlay/ovmanager/ovmanager.go b/drivers/overlay/ovmanager/ovmanager.go index 9198237ac9..830dfb31f3 100644 --- a/drivers/overlay/ovmanager/ovmanager.go +++ b/drivers/overlay/ovmanager/ovmanager.go @@ -210,6 +210,10 @@ func (d *driver) EndpointOperInfo(nid, eid string) (map[string]interface{}, erro return nil, types.NotImplementedErrorf("not implemented") } +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return types.NotImplementedErrorf("not implemented") +} + // Join method is invoked when a Sandbox is attached to an endpoint. func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error { return types.NotImplementedErrorf("not implemented") diff --git a/drivers/remote/driver.go b/drivers/remote/driver.go index 5383d9ee7a..6d334bd85e 100644 --- a/drivers/remote/driver.go +++ b/drivers/remote/driver.go @@ -315,6 +315,10 @@ func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{ return d.call("DiscoverDelete", notif, &api.DiscoveryResponse{}) } +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} + func parseStaticRoutes(r api.JoinResponse) ([]*types.StaticRoute, error) { var routes = make([]*types.StaticRoute, len(r.StaticRoutes)) for i, inRoute := range r.StaticRoutes { diff --git a/drivers/remote/driver_test.go b/drivers/remote/driver_test.go index 8d055732a4..0dcf1fb8ed 100644 --- a/drivers/remote/driver_test.go +++ b/drivers/remote/driver_test.go @@ -132,6 +132,14 @@ func (test *testEndpoint) SetIPAddress(address *net.IPNet) error { return setAddress(&test.address, address) } +func (test *testEndpoint) testEndpoint() string { + return "" +} + +func (test *testEndpoint) SrcName() string { + return "" +} + func setAddress(ifaceAddr *string, address *net.IPNet) error { if *ifaceAddr != "" { return types.ForbiddenErrorf("endpoint interface IP present (%s). Cannot be modified with (%s).", *ifaceAddr, address) @@ -527,6 +535,10 @@ func (r *rollbackEndpoint) AddressIPv6() *net.IPNet { return nil } +func (r *rollbackEndpoint) SrcName() string { + return "" +} + func (r *rollbackEndpoint) SetMacAddress(mac net.HardwareAddr) error { return fmt.Errorf("invalid mac") } diff --git a/drivers/windows/windows.go b/drivers/windows/windows.go index aa4c7e5808..6390a9bdb3 100644 --- a/drivers/windows/windows.go +++ b/drivers/windows/windows.go @@ -593,3 +593,7 @@ func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { return nil } + +func (d *driver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} diff --git a/drvregistry/drvregistry_test.go b/drvregistry/drvregistry_test.go index 0b77b6fecb..1ca3afd0a4 100644 --- a/drvregistry/drvregistry_test.go +++ b/drvregistry/drvregistry_test.go @@ -53,6 +53,10 @@ func (m *mockDriver) Leave(nid, eid string) error { return nil } +func (m *mockDriver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} + func (m *mockDriver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { return nil } diff --git a/endpoint.go b/endpoint.go index c4c6a0e1dc..abe7dee7cb 100644 --- a/endpoint.go +++ b/endpoint.go @@ -83,6 +83,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) { epMap["name"] = ep.name epMap["id"] = ep.id epMap["ep_iface"] = ep.iface + epMap["joinInfo"] = ep.joinInfo epMap["exposed_ports"] = ep.exposedPorts if ep.generic != nil { epMap["generic"] = ep.generic @@ -113,6 +114,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) { ib, _ := json.Marshal(epMap["ep_iface"]) json.Unmarshal(ib, &ep.iface) + jb, _ := json.Marshal(epMap["joinInfo"]) + json.Unmarshal(jb, &ep.joinInfo) + tb, _ := json.Marshal(epMap["exposed_ports"]) var tPorts []types.TransportPort json.Unmarshal(tb, &tPorts) @@ -225,6 +229,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error { ep.iface.CopyTo(dstEp.iface) } + if ep.joinInfo != nil { + dstEp.joinInfo = &endpointJoinInfo{} + ep.joinInfo.CopyTo(dstEp.joinInfo) + } + dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts)) copy(dstEp.exposedPorts, ep.exposedPorts) @@ -1045,6 +1054,13 @@ func (ep *endpoint) releaseAddress() { } func (c *controller) cleanupLocalEndpoints() { + // Get used endpoints + eps := make(map[string]interface{}) + for _, sb := range c.sandboxes { + for _, ep := range sb.endpoints { + eps[ep.id] = true + } + } nl, err := c.getNetworksForScope(datastore.LocalScope) if err != nil { log.Warnf("Could not get list of networks during endpoint cleanup: %v", err) @@ -1059,6 +1075,9 @@ func (c *controller) cleanupLocalEndpoints() { } for _, ep := range epl { + if _, ok := eps[ep.id]; ok { + continue + } log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id) if err := ep.Delete(true); err != nil { log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err) diff --git a/endpoint_info.go b/endpoint_info.go index cc7aa17a66..d3fc270d1b 100644 --- a/endpoint_info.go +++ b/endpoint_info.go @@ -139,6 +139,10 @@ func (epi *endpointInterface) CopyTo(dstEpi *endpointInterface) error { return nil } +func (epi *endpointInterface) SrcName() string { + return epi.srcName +} + type endpointJoinInfo struct { gw net.IP gw6 net.IP @@ -383,3 +387,56 @@ func (ep *endpoint) DisableGatewayService() { ep.joinInfo.disableGatewayService = true } + +func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + if epj.gw != nil { + epMap["gw"] = epj.gw.String() + } + if epj.gw6 != nil { + epMap["gw6"] = epj.gw6.String() + } + epMap["disableGatewayService"] = epj.disableGatewayService + epMap["StaticRoutes"] = epj.StaticRoutes + return json.Marshal(epMap) +} + +func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + if err = json.Unmarshal(b, &epMap); err != nil { + return err + } + if v, ok := epMap["gw"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + if v, ok := epMap["gw6"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + epj.disableGatewayService = epMap["disableGatewayService"].(bool) + + var tStaticRoute []types.StaticRoute + if v, ok := epMap["StaticRoutes"]; ok { + tb, _ := json.Marshal(v) + var tStaticRoute []types.StaticRoute + json.Unmarshal(tb, &tStaticRoute) + } + var StaticRoutes []*types.StaticRoute + for _, r := range tStaticRoute { + StaticRoutes = append(StaticRoutes, &r) + } + epj.StaticRoutes = StaticRoutes + + return nil +} + +func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error { + dstEpj.disableGatewayService = epj.disableGatewayService + dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes)) + copy(dstEpj.StaticRoutes, epj.StaticRoutes) + dstEpj.gw = types.GetIPCopy(epj.gw) + dstEpj.gw = types.GetIPCopy(epj.gw6) + return nil +} diff --git a/iptables/iptables.go b/iptables/iptables.go index f6ddaed775..0ae5fb2645 100644 --- a/iptables/iptables.go +++ b/iptables/iptables.go @@ -347,6 +347,33 @@ func existsRaw(table Table, chain string, rule ...string) bool { // Raw calls 'iptables' system command, passing supplied arguments. func Raw(args ...string) ([]byte, error) { + // check if the rule exists + var ( + doCheck bool + ) + check := make([]string, len(args)) + for i, n := range args { + check[i] = n + if n == "-A" || n == "-I" { + check[i] = "-C" + doCheck = true + } else if n == "-D" || n == "-C" { + doCheck = false + break + } + } + if doCheck { + var err error + if firewalldRunning { + _, err = Passthrough(Iptables, check...) + } else { + _, err = raw(check...) + } + if err == nil { + return nil, nil + } + } + if firewalldRunning { output, err := Passthrough(Iptables, args...) if err == nil || !strings.Contains(err.Error(), "was not provided by any .service files") { diff --git a/libnetwork_internal_test.go b/libnetwork_internal_test.go index 000ba7e5e6..0e65932cfe 100644 --- a/libnetwork_internal_test.go +++ b/libnetwork_internal_test.go @@ -283,7 +283,8 @@ func compareAddresses(a, b map[string]*net.IPNet) bool { } func TestAuxAddresses(t *testing.T) { - c, err := New() + old := make(map[string]interface{}) + c, _, err := New(old) if err != nil { t.Fatal(err) } @@ -319,7 +320,8 @@ func TestAuxAddresses(t *testing.T) { } func TestSRVServiceQuery(t *testing.T) { - c, err := New() + old := make(map[string]interface{}) + c, _, err := New(old) if err != nil { t.Fatal(err) } @@ -424,7 +426,8 @@ func TestIpamReleaseOnNetDriverFailures(t *testing.T) { } cfgOptions, err := OptionBoltdbWithRandomDBFile() - c, err := New(cfgOptions...) + old := make(map[string]interface{}) + c, _, err := New(old, cfgOptions...) if err != nil { t.Fatal(err) } @@ -514,6 +517,10 @@ func (b *badDriver) EndpointOperInfo(nid, eid string) (map[string]interface{}, e func (b *badDriver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo, options map[string]interface{}) error { return fmt.Errorf("I will not allow any join") } + +func (b *badDriver) Restore(nid, eid string, sboxKey string, ifInfo driverapi.InterfaceInfo, options map[string]interface{}) error { + return nil +} func (b *badDriver) Leave(nid, eid string) error { return nil } diff --git a/libnetwork_test.go b/libnetwork_test.go index b63ff0f801..1d57765153 100644 --- a/libnetwork_test.go +++ b/libnetwork_test.go @@ -76,7 +76,8 @@ func createController() error { if err != nil { return err } - controller, err = libnetwork.New(append(cfgOptions, config.OptionDriverConfig(bridgeNetType, genericOption))...) + old := make(map[string]interface{}) + controller, _, err = libnetwork.New(old, append(cfgOptions, config.OptionDriverConfig(bridgeNetType, genericOption))...) if err != nil { return err } @@ -2044,7 +2045,8 @@ func TestInvalidRemoteDriver(t *testing.T) { t.Fatal(err) } - ctrlr, err := libnetwork.New() + old := make(map[string]interface{}) + ctrlr, _, err := libnetwork.New(old) if err != nil { t.Fatal(err) } diff --git a/osl/namespace_linux.go b/osl/namespace_linux.go index 07b725c290..b52141ab7a 100644 --- a/osl/namespace_linux.go +++ b/osl/namespace_linux.go @@ -2,10 +2,13 @@ package osl import ( "fmt" + "io/ioutil" "net" "os" "os/exec" "runtime" + "strconv" + "strings" "sync" "syscall" "time" @@ -132,6 +135,39 @@ func GC() { // container id. func GenerateKey(containerID string) string { maxLen := 12 + // Read sandbox key from host for overlay + if strings.HasPrefix(containerID, "-") { + var ( + index int + indexStr string + tmpkey string + ) + dir, err := ioutil.ReadDir(prefix) + if err != nil { + return "" + } + + for _, v := range dir { + id := v.Name() + if strings.HasSuffix(id, containerID[:maxLen-1]) { + indexStr = strings.TrimSuffix(id, containerID[:maxLen-1]) + tmpindex, err := strconv.Atoi(indexStr) + if err != nil { + return "" + } + if tmpindex > index { + index = tmpindex + tmpkey = id + } + + } + } + containerID = tmpkey + if containerID == "" { + return "" + } + } + if len(containerID) < maxLen { maxLen = len(containerID) } @@ -150,6 +186,15 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) { return &networkNamespace{path: key, isDefault: !osCreate}, nil } +// NewNullSandbox restore a sandbox +func NewNullSandbox(key string) Sandbox { + var isDefault bool + if key == "default" { + isDefault = true + } + return &networkNamespace{path: key, isDefault: isDefault} +} + func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter { return n } @@ -321,3 +366,111 @@ func (n *networkNamespace) Destroy() error { addToGarbagePaths(n.path) return nil } + +// Restore restore the network namespace +// TODO: read this information from the netnamespace maybe a better choice ? +func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error { + // restore interfaces + for name, opts := range ifsopt { + if !strings.Contains(name, "+") { + return fmt.Errorf("wrong iface name in restore osl sandbox interface") + } + seps := strings.Split(name, "+") + srcName := seps[0] + dstPrefix := seps[1] + i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} + i.processInterfaceOptions(opts...) + if i.master != "" { + i.dstMaster = n.findDst(i.master, true) + if i.dstMaster == "" { + return fmt.Errorf("could not find an appropriate master %q for %q", + i.master, i.srcName) + } + } + + if n.isDefault { + i.dstName = i.srcName + } else { + // due to the docker network connect/disconnect, so the dstName should + // restore from the namespace + err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error { + ifaces, err := net.Interfaces() + if err != nil { + return err + } + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + return err + } + if strings.HasPrefix(iface.Name, "vxlan") { + if i.dstName == "vxlan" { + i.dstName = iface.Name + break + } + } + + // find the interface name by ip + if i.address != nil { + for _, addr := range addrs { + if addr.String() == i.address.String() { + i.dstName = iface.Name + break + } + continue + } + if i.dstName == iface.Name { + break + } + } + // This is to find the interface name of the pair in overlay sandbox + if strings.HasPrefix(iface.Name, "veth") { + if i.master != "" && i.dstName == "veth" { + i.dstName = iface.Name + } + } + } + return nil + }) + if err != nil { + return err + } + var index int + indexStr := strings.TrimPrefix(i.dstName, dstPrefix) + if indexStr != "" { + index, err = strconv.Atoi(indexStr) + if err != nil { + return err + } + } + index++ + n.Lock() + if index > n.nextIfIndex { + n.nextIfIndex = index + } + n.iFaces = append(n.iFaces, i) + n.Unlock() + } + } + + // restore routes + for _, r := range routes { + n.Lock() + n.staticRoutes = append(n.staticRoutes, r) + n.Unlock() + } + + // restore gateway + if len(gw) > 0 { + n.Lock() + n.gw = gw + n.Unlock() + } + + if len(gw6) > 0 { + n.Lock() + n.gwv6 = gw6 + n.Unlock() + } + return nil +} diff --git a/osl/namespace_windows.go b/osl/namespace_windows.go index 912d4a2e9f..ea344c2285 100644 --- a/osl/namespace_windows.go +++ b/osl/namespace_windows.go @@ -19,6 +19,11 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) { return nil, nil } +// NewNullSandbox restore sandbox +func NewNullSandbox(key string) Sandbox { + return nil +} + func GetSandboxForExternalKey(path string, key string) (Sandbox, error) { return nil, nil } diff --git a/osl/neigh_linux.go b/osl/neigh_linux.go index a221e712da..6553f1c89f 100644 --- a/osl/neigh_linux.go +++ b/osl/neigh_linux.go @@ -130,9 +130,7 @@ func (n *networkNamespace) AddNeighbor(dstIP net.IP, dstMac net.HardwareAddr, op if err := netlink.NeighSet(nlnh); err != nil { return fmt.Errorf("could not add neighbor entry: %v", err) } - n.neighbors = append(n.neighbors, nh) - return nil }) } diff --git a/osl/sandbox.go b/osl/sandbox.go index db49d43dce..072c443421 100644 --- a/osl/sandbox.go +++ b/osl/sandbox.go @@ -58,6 +58,9 @@ type Sandbox interface { // Destroy the sandbox Destroy() error + + // restore sandbox + Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error } // NeighborOptionSetter interfaces defines the option setter methods for interface options diff --git a/portmapper/mapper.go b/portmapper/mapper.go index d125fa8d4b..62112528e4 100644 --- a/portmapper/mapper.go +++ b/portmapper/mapper.go @@ -62,11 +62,11 @@ func (pm *PortMapper) SetIptablesChain(c *iptables.ChainInfo, bridgeName string) // Map maps the specified container transport address to the host's network address and transport port func (pm *PortMapper) Map(container net.Addr, hostIP net.IP, hostPort int, useProxy bool) (host net.Addr, err error) { - return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy) + return pm.MapRange(container, hostIP, hostPort, hostPort, useProxy, false) } // MapRange maps the specified container transport address to the host's network address and transport port range -func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy bool) (host net.Addr, err error) { +func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, hostPortEnd int, useProxy, restore bool) (host net.Addr, err error) { pm.lock.Lock() defer pm.lock.Unlock() @@ -90,7 +90,15 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } if useProxy { - m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) + if restore { + m.userlandProxy, err = newProxyRestored(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) + if err != nil { + logrus.Warnf("failed to restore old userland proxy, re-create userland proxy") + } + } + if m.userlandProxy == nil { + m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.TCPAddr).IP, container.(*net.TCPAddr).Port) + } } else { m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort) } @@ -107,7 +115,15 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } if useProxy { - m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port) + if restore { + m.userlandProxy, err = newProxyRestored(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port) + if err != nil { + logrus.Warnf("failed to restore old userland proxy, re-create userland proxy") + } + } + if m.userlandProxy == nil { + m.userlandProxy = newProxy(proto, hostIP, allocatedHostPort, container.(*net.UDPAddr).IP, container.(*net.UDPAddr).Port) + } } else { m.userlandProxy = newDummyProxy(proto, hostIP, allocatedHostPort) } @@ -128,6 +144,8 @@ func (pm *PortMapper) MapRange(container net.Addr, hostIP net.IP, hostPortStart, } containerIP, containerPort := getIPAndPort(m.container) + + // on networking restore we still try to insert the iptalbes just to make sure all the iptables are exists. if err := pm.forward(iptables.Append, m.proto, hostIP, allocatedHostPort, containerIP.String(), containerPort); err != nil { return nil, err } diff --git a/portmapper/proxy.go b/portmapper/proxy.go index ddde2744c2..62c2694eb9 100644 --- a/portmapper/proxy.go +++ b/portmapper/proxy.go @@ -106,9 +106,6 @@ func newProxyCommand(proto string, hostIP net.IP, hostPort int, containerIP net. cmd: &exec.Cmd{ Path: reexec.Self(), Args: args, - SysProcAttr: &syscall.SysProcAttr{ - Pdeathsig: syscall.SIGTERM, // send a sigterm to the proxy if the daemon process dies - }, }, } } diff --git a/portmapper/restore_proxy.go b/portmapper/restore_proxy.go new file mode 100644 index 0000000000..5c7b076259 --- /dev/null +++ b/portmapper/restore_proxy.go @@ -0,0 +1,77 @@ +//+build !windows + +package portmapper + +import ( + "fmt" + "net" + "os" + "os/exec" + "strconv" + "strings" + + "github.com/Sirupsen/logrus" +) + +type proxyRestored struct { + process *os.Process +} + +func newProxyRestored(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) { + str := fmt.Sprintf("%s/%s/%d/%s/%d", proto, hostIP.String(), hostPort, containerIP.String(), containerPort) + pid, err := getProcessPid(str) + if err != nil { + return nil, err + } + logrus.Debugf("restore old userland proxy 'pid=%d' for %s ", pid, str) + process, err := os.FindProcess(pid) + if err != nil { + return nil, err + } + return &proxyRestored{process: process}, nil +} + +func (p *proxyRestored) Start() error { + return nil +} + +func (p *proxyRestored) Stop() error { + p.process.Kill() + return nil +} + +func getProcessPid(s string) (int, error) { + psArgs := "-ef" + output, err := exec.Command("ps", strings.Split(psArgs, " ")...).Output() + if err != nil { + return -1, fmt.Errorf("Error running ps: %v", err) + } + lines := strings.Split(string(output), "\n") + pidIndex := -1 + cmdIndex := -1 + for i, name := range strings.Fields(lines[0]) { + if name == "PID" { + pidIndex = i + } + if name == "CMD" { + cmdIndex = i + } + } + for _, line := range lines[1:] { + if len(line) == 0 { + continue + } + fields := strings.Fields(line) + if strings.Compare(fields[cmdIndex], userlandProxyCommandName) == 0 { + str := fmt.Sprintf("%s/%s/%s/%s/%s", fields[cmdIndex+2], fields[cmdIndex+4], fields[cmdIndex+6], fields[cmdIndex+8], fields[cmdIndex+10]) + if strings.Compare(str, s) == 0 { + p, err := strconv.Atoi(fields[pidIndex]) + if err != nil { + return -1, fmt.Errorf("Unexpected pid '%s': %s", fields[pidIndex], err) + } + return p, nil + } + } + } + return -1, fmt.Errorf("no such proxy") +} diff --git a/portmapper/restore_proxy_windows.go b/portmapper/restore_proxy_windows.go new file mode 100644 index 0000000000..5328faa20e --- /dev/null +++ b/portmapper/restore_proxy_windows.go @@ -0,0 +1,22 @@ +//+build windows + +package portmapper + +import ( + "net" +) + +type proxyRestored struct { +} + +func newProxyRestored(proto string, hostIP net.IP, hostPort int, containerIP net.IP, containerPort int) (userlandProxy, error) { + return nil, nil +} + +func (p *proxyRestored) Start() error { + return nil +} + +func (p *proxyRestored) Stop() error { + return nil +} diff --git a/resolver_unix.go b/resolver_unix.go index 2b3734fbac..cec2c7d493 100644 --- a/resolver_unix.go +++ b/resolver_unix.go @@ -19,6 +19,13 @@ func init() { reexec.Register("setup-resolver", reexecSetupResolver) } +const ( + // outputChain used for docker embed dns + outputChain = "DOCKER_OUTPUT" + //postroutingchain used for docker embed dns + postroutingchain = "DOCKER_POSTROUTING" +) + func reexecSetupResolver() { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -31,10 +38,10 @@ func reexecSetupResolver() { _, ipPort, _ := net.SplitHostPort(os.Args[2]) _, tcpPort, _ := net.SplitHostPort(os.Args[3]) rules := [][]string{ - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, } f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0) @@ -50,6 +57,23 @@ func reexecSetupResolver() { os.Exit(3) } + // insert outputChain and postroutingchain + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain) + } + + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + } + for _, rule := range rules { if iptables.RawCombinedOutputNative(rule...) != nil { log.Errorf("setting up rule failed, %v", rule) diff --git a/sandbox.go b/sandbox.go index 36cbac6627..22ad85f1c3 100644 --- a/sandbox.go +++ b/sandbox.go @@ -696,6 +696,51 @@ func (sb *sandbox) releaseOSSbox() { osSbox.Destroy() } +func (sb *sandbox) restoreSandbox(option []SandboxOption) error { + var ( + routes []*types.StaticRoute + ) + + // restore sb.config + sb.processOptions(option...) + sb.restorePath() + + // restore osl sandbox + Ifaces := make(map[string][]osl.IfaceOption) + ifaceOptions := make([][]osl.IfaceOption, len(sb.endpoints)) + for j, ep := range sb.endpoints { + ep.Lock() + joinInfo := ep.joinInfo + i := ep.iface + ep.Unlock() + ifaceOptions[j] = append(ifaceOptions[j], sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) + if i.addrv6 != nil && i.addrv6.IP.To16() != nil { + ifaceOptions[j] = append(ifaceOptions[j], sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) + } + if i.mac != nil { + ifaceOptions[j] = append(ifaceOptions[j], sb.osSbox.InterfaceOptions().MacAddress(i.mac)) + } + Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions[j] + if joinInfo != nil { + for _, r := range joinInfo.StaticRoutes { + routes = append(routes, r) + } + } + if ep.needResolver() { + sb.startResolver() + } + } + gwep := sb.getGatewayEndpoint() + joinInfo := gwep.joinInfo + + // restore osl sandbox + err := sb.osSbox.Restore(Ifaces, routes, joinInfo.gw, joinInfo.gw6) + if err != nil { + return err + } + return nil +} + func (sb *sandbox) populateNetworkResources(ep *endpoint) error { sb.Lock() if sb.osSbox == nil { diff --git a/sandbox_dns_unix.go b/sandbox_dns_unix.go index 8d59e3d66a..b0edd6a963 100644 --- a/sandbox_dns_unix.go +++ b/sandbox_dns_unix.go @@ -139,6 +139,17 @@ func (sb *sandbox) updateParentHosts() error { return nil } +func (sb *sandbox) restorePath() { + if sb.config.resolvConfPath == "" { + sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf" + } + sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash" + if sb.config.hostsPath == "" { + sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts" + } + +} + func (sb *sandbox) setupDNS() error { var newRC *resolvconf.File diff --git a/sandbox_dns_windows.go b/sandbox_dns_windows.go index ef90ddaeef..f2f58d5b98 100644 --- a/sandbox_dns_windows.go +++ b/sandbox_dns_windows.go @@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error { return nil } +func (sb *sandbox) restorePath() { +} + func (sb *sandbox) updateHostsFile(ifaceIP string) error { return nil } diff --git a/sandbox_store.go b/sandbox_store.go index ae5ddc1566..0e6a5520c3 100644 --- a/sandbox_store.go +++ b/sandbox_store.go @@ -20,12 +20,13 @@ type epState struct { } type sbState struct { - ID string - Cid string - c *controller - dbIndex uint64 - dbExists bool - Eps []epState + ID string + Cid string + c *controller + dbIndex uint64 + dbExists bool + Eps []epState + EpPriority map[string]int } func (sbs *sbState) Key() []string { @@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error { dstSbs.Cid = sbs.Cid dstSbs.dbIndex = sbs.dbIndex dstSbs.dbExists = sbs.dbExists + dstSbs.EpPriority = sbs.EpPriority for _, eps := range sbs.Eps { dstSbs.Eps = append(dstSbs.Eps, eps) @@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string { func (sb *sandbox) storeUpdate() error { sbs := &sbState{ - c: sb.controller, - ID: sb.id, - Cid: sb.containerID, + c: sb.controller, + ID: sb.id, + Cid: sb.containerID, + EpPriority: sb.epPriority, } retry: @@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error { return sb.controller.deleteFromStore(sbs) } -func (c *controller) sandboxCleanup() { +func (c *controller) sandboxCleanup(sbids map[string]interface{}) { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") @@ -198,7 +201,7 @@ func (c *controller) sandboxCleanup() { dbExists: true, } - sb.osSbox, err = osl.NewSandbox(sb.Key(), true) + sb.osSbox = osl.NewNullSandbox(sb.Key()) if err != nil { logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err) continue @@ -225,10 +228,11 @@ func (c *controller) sandboxCleanup() { heap.Push(&sb.endpoints, ep) } - - logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) - if err := sb.delete(true); err != nil { - logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + if _, ok := sbids[sb.containerID]; !ok { + logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) + if err := sb.delete(true); err != nil { + logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + } } } } diff --git a/sandbox_test.go b/sandbox_test.go index 6ce245d767..a4cc82ec8a 100644 --- a/sandbox_test.go +++ b/sandbox_test.go @@ -23,7 +23,8 @@ func getTestEnv(t *testing.T, empty bool) (NetworkController, Network, Network) if err != nil { t.Fatal(err) } - c, err := New(append(cfgOptions, config.OptionDriverConfig(netType, genericOption))...) + old := make(map[string]interface{}) + c, _, err := New(old, append(cfgOptions, config.OptionDriverConfig(netType, genericOption))...) if err != nil { t.Fatal(err) } diff --git a/store_test.go b/store_test.go index 7741004423..57d7969193 100644 --- a/store_test.go +++ b/store_test.go @@ -28,7 +28,9 @@ func testNewController(t *testing.T, provider, url string) (NetworkController, e } cfgOptions = append(cfgOptions, config.OptionKVProvider(provider)) cfgOptions = append(cfgOptions, config.OptionKVProviderURL(url)) - return New(cfgOptions...) + old := make(map[string]interface{}) + controller, _, err := New(old, cfgOptions...) + return controller, err } func TestBoltdbBackend(t *testing.T) { @@ -51,7 +53,8 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con genericOption[netlabel.GenericData] = driverOptions cfgOptions = append(cfgOptions, config.OptionDriverConfig("host", genericOption)) - ctrl, err := New(cfgOptions...) + old := make(map[string]interface{}) + ctrl, _, err := New(old, cfgOptions...) if err != nil { t.Fatalf("Error new controller: %v", err) } @@ -73,7 +76,7 @@ func testLocalBackend(t *testing.T, provider, url string, storeConfig *store.Con store.Close() // test restore of local store - ctrl, err = New(cfgOptions...) + ctrl, _, err = New(old, cfgOptions...) if err != nil { t.Fatalf("Error creating controller: %v", err) } @@ -87,7 +90,8 @@ func TestNoPersist(t *testing.T) { if err != nil { t.Fatalf("Error creating random boltdb file : %v", err) } - ctrl, err := New(cfgOptions...) + old := make(map[string]interface{}) + ctrl, _, err := New(old, cfgOptions...) if err != nil { t.Fatalf("Error new controller: %v", err) } @@ -131,13 +135,14 @@ func TestMultipleControllersWithSameStore(t *testing.T) { if err != nil { t.Fatalf("Error getting random boltdb configs %v", err) } - ctrl1, err := New(cfgOptions...) + old := make(map[string]interface{}) + ctrl1, _, err := New(old, cfgOptions...) if err != nil { t.Fatalf("Error new controller: %v", err) } defer ctrl1.Stop() // Use the same boltdb file without closing the previous controller - _, err = New(cfgOptions...) + _, _, err = New(old, cfgOptions...) if err != nil { t.Fatalf("Local store must support concurrent controllers") }