diff --git a/manager/allocator/networkallocator/networkallocator.go b/manager/allocator/networkallocator/networkallocator.go index df79849f04..39b13d0007 100644 --- a/manager/allocator/networkallocator/networkallocator.go +++ b/manager/allocator/networkallocator/networkallocator.go @@ -573,6 +573,7 @@ func (na *NetworkAllocator) releaseEndpoints(networks []*api.NetworkAttachment) // allocate virtual IP for a single endpoint attachment of the service. func (na *NetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error { + var opts map[string]string localNet := na.getNetwork(vip.NetworkID) if localNet == nil { return errors.New("networkallocator: could not find local network state") @@ -602,9 +603,13 @@ func (na *NetworkAllocator) allocateVIP(vip *api.Endpoint_VirtualIP) error { return err } } + if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil { + // set ipam allocation method to serial + opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options) + } for _, poolID := range localNet.pools { - ip, _, err := ipam.RequestAddress(poolID, addr, nil) + ip, _, err := ipam.RequestAddress(poolID, addr, opts) if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange { return errors.Wrap(err, "could not allocate VIP from IPAM") } @@ -656,6 +661,7 @@ func (na *NetworkAllocator) deallocateVIP(vip *api.Endpoint_VirtualIP) error { // allocate the IP addresses for a single network attachment of the task. func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) error { var ip *net.IPNet + var opts map[string]string ipam, _, _, err := na.resolveIPAM(nAttach.Network) if err != nil { @@ -685,11 +691,16 @@ func (na *NetworkAllocator) allocateNetworkIPs(nAttach *api.NetworkAttachment) e } } } + // Set the ipam options if the network has an ipam driver. + if localNet.nw.IPAM != nil && localNet.nw.IPAM.Driver != nil { + // set ipam allocation method to serial + opts = setIPAMSerialAlloc(localNet.nw.IPAM.Driver.Options) + } for _, poolID := range localNet.pools { var err error - ip, _, err = ipam.RequestAddress(poolID, addr, nil) + ip, _, err = ipam.RequestAddress(poolID, addr, opts) if err != nil && err != ipamapi.ErrNoAvailableIPs && err != ipamapi.ErrIPOutOfRange { return errors.Wrap(err, "could not allocate IP from IPAM") } @@ -917,8 +928,16 @@ func (na *NetworkAllocator) allocatePools(n *api.Network) (map[string]string, er } gwIP.IP = ip } + if dOptions == nil { + dOptions = make(map[string]string) + } + dOptions[ipamapi.RequestAddressType] = netlabel.Gateway + // set ipam allocation method to serial + dOptions = setIPAMSerialAlloc(dOptions) + defer delete(dOptions, ipamapi.RequestAddressType) + if ic.Gateway != "" || gwIP == nil { - gwIP, _, err = ipam.RequestAddress(poolID, net.ParseIP(ic.Gateway), map[string]string{ipamapi.RequestAddressType: netlabel.Gateway}) + gwIP, _, err = ipam.RequestAddress(poolID, net.ParseIP(ic.Gateway), dOptions) if err != nil { // Rollback by releasing all the resources allocated so far. releasePools(ipam, ipamConfigs[:i], pools) @@ -1012,3 +1031,14 @@ func IsBuiltInDriver(name string) bool { } return false } + +// setIPAMSerialAlloc sets the ipam allocation method to serial +func setIPAMSerialAlloc(opts map[string]string) map[string]string { + if opts == nil { + opts = make(map[string]string) + } + if _, ok := opts[ipamapi.AllocSerialPrefix]; !ok { + opts[ipamapi.AllocSerialPrefix] = "true" + } + return opts +} diff --git a/manager/allocator/networkallocator/networkallocator_test.go b/manager/allocator/networkallocator/networkallocator_test.go index ac6b0bfa3a..13d9b043e8 100644 --- a/manager/allocator/networkallocator/networkallocator_test.go +++ b/manager/allocator/networkallocator/networkallocator_test.go @@ -771,8 +771,8 @@ func TestServiceAddRemovePortsIngressMode(t *testing.T) { assert.Len(t, s.Endpoint.Ports, 0) assert.Len(t, s.Endpoint.VirtualIPs, 0) - // Publish port again and ensure VIP is the same that was deallocated - // and there is no leak. + // Publish port again and ensure VIP is not the same that was deallocated. + // Since IP allocation is serial we should receive the next available IP. s.Spec.Endpoint.Ports = append(s.Spec.Endpoint.Ports, &api.PortConfig{Name: "some_tcp", TargetPort: 1234, PublishedPort: 1234, @@ -785,7 +785,7 @@ func TestServiceAddRemovePortsIngressMode(t *testing.T) { assert.Len(t, s.Endpoint.Ports, 1) assert.Equal(t, uint32(1234), s.Endpoint.Ports[0].PublishedPort) assert.Len(t, s.Endpoint.VirtualIPs, 1) - assert.Equal(t, allocatedVIP, s.Endpoint.VirtualIPs[0].Addr) + assert.NotEqual(t, allocatedVIP, s.Endpoint.VirtualIPs[0].Addr) } func TestServiceUpdate(t *testing.T) { diff --git a/manager/allocator/networkallocator/portallocator.go b/manager/allocator/networkallocator/portallocator.go index af8126ea25..24614b17f0 100644 --- a/manager/allocator/networkallocator/portallocator.go +++ b/manager/allocator/networkallocator/portallocator.go @@ -382,7 +382,7 @@ func (ps *portSpace) allocate(p *api.PortConfig) (err error) { } // Check out an arbitrary port from dynamic port space. - swarmPort, err := ps.dynamicPortSpace.GetID() + swarmPort, err := ps.dynamicPortSpace.GetID(true) if err != nil { return } diff --git a/vendor.conf b/vendor.conf index 52fa7aafe5..0b45cfb292 100644 --- a/vendor.conf +++ b/vendor.conf @@ -23,7 +23,7 @@ github.com/docker/go-connections 34b5052da6b11e27f5f2e357b38b571ddddd3928 github.com/docker/go-events 37d35add5005832485c0225ec870121b78fcff1c github.com/docker/go-units 954fed01cc617c55d838fa2230073f2cb17386c8 github.com/docker/libkv 9fd56606e928ff1f309808f5d5a0b7a2ef73f9a8 -github.com/docker/libnetwork 37e20af882e13dd01ade3658b7aabdae3412118b +github.com/docker/libnetwork 25704095177f49bc28e8e6f616936f7073d7b56d github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a github.com/opencontainers/runc 8e8d01d38d7b4fb0a35bf89b72bc3e18c98882d7 github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb diff --git a/vendor/github.com/docker/libnetwork/bitseq/sequence.go b/vendor/github.com/docker/libnetwork/bitseq/sequence.go index 86cf69b34f..a209d1ec7c 100644 --- a/vendor/github.com/docker/libnetwork/bitseq/sequence.go +++ b/vendor/github.com/docker/libnetwork/bitseq/sequence.go @@ -41,6 +41,7 @@ type Handle struct { id string dbIndex uint64 dbExists bool + curr uint64 store datastore.DataStore sync.Mutex } @@ -193,26 +194,27 @@ func (h *Handle) getCopy() *Handle { dbIndex: h.dbIndex, dbExists: h.dbExists, store: h.store, + curr: h.curr, } } // SetAnyInRange atomically sets the first unset bit in the specified range in the sequence and returns the corresponding ordinal -func (h *Handle) SetAnyInRange(start, end uint64) (uint64, error) { +func (h *Handle) SetAnyInRange(start, end uint64, serial bool) (uint64, error) { if end < start || end >= h.bits { return invalidPos, fmt.Errorf("invalid bit range [%d, %d]", start, end) } if h.Unselected() == 0 { return invalidPos, ErrNoBitAvailable } - return h.set(0, start, end, true, false) + return h.set(0, start, end, true, false, serial) } // SetAny atomically sets the first unset bit in the sequence and returns the corresponding ordinal -func (h *Handle) SetAny() (uint64, error) { +func (h *Handle) SetAny(serial bool) (uint64, error) { if h.Unselected() == 0 { return invalidPos, ErrNoBitAvailable } - return h.set(0, 0, h.bits-1, true, false) + return h.set(0, 0, h.bits-1, true, false, serial) } // Set atomically sets the corresponding bit in the sequence @@ -220,7 +222,7 @@ func (h *Handle) Set(ordinal uint64) error { if err := h.validateOrdinal(ordinal); err != nil { return err } - _, err := h.set(ordinal, 0, 0, false, false) + _, err := h.set(ordinal, 0, 0, false, false, false) return err } @@ -229,7 +231,7 @@ func (h *Handle) Unset(ordinal uint64) error { if err := h.validateOrdinal(ordinal); err != nil { return err } - _, err := h.set(ordinal, 0, 0, false, true) + _, err := h.set(ordinal, 0, 0, false, true, false) return err } @@ -298,7 +300,7 @@ func (h *Handle) CheckConsistency() error { } // set/reset the bit -func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64, error) { +func (h *Handle) set(ordinal, start, end uint64, any bool, release bool, serial bool) (uint64, error) { var ( bitPos uint64 bytePos uint64 @@ -308,6 +310,7 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64 for { var store datastore.DataStore + curr := uint64(0) h.Lock() store = h.store h.Unlock() @@ -318,15 +321,18 @@ func (h *Handle) set(ordinal, start, end uint64, any bool, release bool) (uint64 } h.Lock() + if serial { + curr = h.curr + } // Get position if available if release { bytePos, bitPos = ordinalToPos(ordinal) } else { if any { - bytePos, bitPos, err = getFirstAvailable(h.head, start) + bytePos, bitPos, err = getAvailableFromCurrent(h.head, start, curr, end) ret = posToOrdinal(bytePos, bitPos) - if end < ret { - err = ErrNoBitAvailable + if err == nil { + h.curr = ret + 1 } } else { bytePos, bitPos, err = checkIfAvailable(h.head, ordinal) @@ -497,7 +503,10 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { // Derive the this sequence offsets byteOffset := byteStart - inBlockBytePos bitOffset := inBlockBytePos*8 + bitStart - + var firstOffset uint64 + if current == head { + firstOffset = byteOffset + } for current != nil { if current.block != blockMAX { bytePos, bitPos, err := current.getAvailableBit(bitOffset) @@ -505,12 +514,36 @@ func getFirstAvailable(head *sequence, start uint64) (uint64, uint64, error) { } // Moving to next block: Reset bit offset. bitOffset = 0 - byteOffset += current.count * blockBytes + byteOffset += (current.count * blockBytes) - firstOffset + firstOffset = 0 current = current.next } return invalidPos, invalidPos, ErrNoBitAvailable } +// getAvailableFromCurrent will look for available ordinal from the current ordinal. +// If none found then it will loop back to the start to check of the available bit. +// This can be further optimized to check from start till curr in case of a rollover +func getAvailableFromCurrent(head *sequence, start, curr, end uint64) (uint64, uint64, error) { + var bytePos, bitPos uint64 + if curr != 0 && curr > start { + bytePos, bitPos, _ = getFirstAvailable(head, curr) + ret := posToOrdinal(bytePos, bitPos) + if end < ret { + goto begin + } + return bytePos, bitPos, nil + } + +begin: + bytePos, bitPos, _ = getFirstAvailable(head, start) + ret := posToOrdinal(bytePos, bitPos) + if end < ret { + return invalidPos, invalidPos, ErrNoBitAvailable + } + return bytePos, bitPos, nil +} + // checkIfAvailable checks if the bit correspondent to the specified ordinal is unset // If the ordinal is beyond the sequence limits, a negative response is returned func checkIfAvailable(head *sequence, ordinal uint64) (uint64, uint64, error) { diff --git a/vendor/github.com/docker/libnetwork/bitseq/store.go b/vendor/github.com/docker/libnetwork/bitseq/store.go index 5448927eb1..cdb7f04264 100644 --- a/vendor/github.com/docker/libnetwork/bitseq/store.go +++ b/vendor/github.com/docker/libnetwork/bitseq/store.go @@ -87,6 +87,7 @@ func (h *Handle) CopyTo(o datastore.KVObject) error { dstH.dbIndex = h.dbIndex dstH.dbExists = h.dbExists dstH.store = h.store + dstH.curr = h.curr dstH.Unlock() return nil diff --git a/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go b/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go index 3c3c0bb257..51310ebfad 100644 --- a/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go +++ b/vendor/github.com/docker/libnetwork/drivers/overlay/ovmanager/ovmanager.go @@ -165,7 +165,7 @@ func (n *network) obtainVxlanID(s *subnet) error { n.Unlock() if vni == 0 { - vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd) + vni, err = n.driver.vxlanIdm.GetIDInRange(vxlanIDStart, vxlanIDEnd, true) if err != nil { return err } diff --git a/vendor/github.com/docker/libnetwork/idm/idm.go b/vendor/github.com/docker/libnetwork/idm/idm.go index 7e449a0dc8..d5843d4a58 100644 --- a/vendor/github.com/docker/libnetwork/idm/idm.go +++ b/vendor/github.com/docker/libnetwork/idm/idm.go @@ -34,11 +34,11 @@ func New(ds datastore.DataStore, id string, start, end uint64) (*Idm, error) { } // GetID returns the first available id in the set -func (i *Idm) GetID() (uint64, error) { +func (i *Idm) GetID(serial bool) (uint64, error) { if i.handle == nil { return 0, errors.New("ID set is not initialized") } - ordinal, err := i.handle.SetAny() + ordinal, err := i.handle.SetAny(serial) return i.start + ordinal, err } @@ -56,7 +56,7 @@ func (i *Idm) GetSpecificID(id uint64) error { } // GetIDInRange returns the first available id in the set within a [start,end] range -func (i *Idm) GetIDInRange(start, end uint64) (uint64, error) { +func (i *Idm) GetIDInRange(start, end uint64, serial bool) (uint64, error) { if i.handle == nil { return 0, errors.New("ID set is not initialized") } @@ -65,7 +65,7 @@ func (i *Idm) GetIDInRange(start, end uint64) (uint64, error) { return 0, errors.New("Requested range does not belong to the set") } - ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start) + ordinal, err := i.handle.SetAnyInRange(start-i.start, end-i.start, serial) return i.start + ordinal, err } diff --git a/vendor/github.com/docker/libnetwork/ipam/allocator.go b/vendor/github.com/docker/libnetwork/ipam/allocator.go index 4243d57a74..e72362b7c9 100644 --- a/vendor/github.com/docker/libnetwork/ipam/allocator.go +++ b/vendor/github.com/docker/libnetwork/ipam/allocator.go @@ -457,7 +457,15 @@ func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[s return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v", k.String(), prefAddress, poolID, err) } - ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range) + // In order to request for a serial ip address allocation, callers can pass in the option to request + // IP allocation serially or first available IP in the subnet + var serial bool + if opts != nil { + if val, ok := opts[ipamapi.AllocSerialPrefix]; ok { + serial = (val == "true") + } + } + ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial) if err != nil { return nil, nil, err } @@ -522,7 +530,7 @@ func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error { return bm.Unset(ipToUint64(h)) } -func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange) (net.IP, error) { +func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) { var ( ordinal uint64 err error @@ -535,7 +543,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres return nil, ipamapi.ErrNoAvailableIPs } if ipr == nil && prefAddress == nil { - ordinal, err = bitmask.SetAny() + ordinal, err = bitmask.SetAny(serial) } else if prefAddress != nil { hostPart, e := types.GetHostPartIP(prefAddress, base.Mask) if e != nil { @@ -544,7 +552,7 @@ func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddres ordinal = ipToUint64(types.GetMinimalIP(hostPart)) err = bitmask.Set(ordinal) } else { - ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End) + ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial) } switch err { @@ -579,7 +587,7 @@ func (a *Allocator) DumpDatabase() string { s = fmt.Sprintf("\n\n%s Config", as) aSpace.Lock() for k, config := range aSpace.subnets { - s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n%v: %v", k, config)) + s += fmt.Sprintf("\n%v: %v", k, config) if config.Range == nil { a.retrieveBitmask(k, config.Pool) } @@ -589,7 +597,7 @@ func (a *Allocator) DumpDatabase() string { s = fmt.Sprintf("%s\n\nBitmasks", s) for k, bm := range a.addresses { - s = fmt.Sprintf("%s%s", s, fmt.Sprintf("\n%s: %s", k, bm)) + s += fmt.Sprintf("\n%s: %s", k, bm) } return s diff --git a/vendor/github.com/docker/libnetwork/ipamapi/labels.go b/vendor/github.com/docker/libnetwork/ipamapi/labels.go new file mode 100644 index 0000000000..e5c7d1cc7e --- /dev/null +++ b/vendor/github.com/docker/libnetwork/ipamapi/labels.go @@ -0,0 +1,10 @@ +package ipamapi + +const ( + // Prefix constant marks the reserved label space for libnetwork + Prefix = "com.docker.network" + + // AllocSerialPrefix constant marks the reserved label space for libnetwork ipam + // allocation ordering.(serial/first available) + AllocSerialPrefix = Prefix + ".ipam.serial" +) diff --git a/vendor/github.com/docker/libnetwork/types/types.go b/vendor/github.com/docker/libnetwork/types/types.go index da113e24cd..164b18096c 100644 --- a/vendor/github.com/docker/libnetwork/types/types.go +++ b/vendor/github.com/docker/libnetwork/types/types.go @@ -129,11 +129,11 @@ func (p *PortBinding) GetCopy() PortBinding { func (p *PortBinding) String() string { ret := fmt.Sprintf("%s/", p.Proto) if p.IP != nil { - ret = fmt.Sprintf("%s%s", ret, p.IP.String()) + ret += p.IP.String() } ret = fmt.Sprintf("%s:%d/", ret, p.Port) if p.HostIP != nil { - ret = fmt.Sprintf("%s%s", ret, p.HostIP.String()) + ret += p.HostIP.String() } ret = fmt.Sprintf("%s:%d", ret, p.HostPort) return ret diff --git a/vendor/github.com/docker/libnetwork/vendor.conf b/vendor/github.com/docker/libnetwork/vendor.conf index 0f1db6d18f..203e1cebf5 100644 --- a/vendor/github.com/docker/libnetwork/vendor.conf +++ b/vendor/github.com/docker/libnetwork/vendor.conf @@ -19,11 +19,11 @@ github.com/docker/libkv 1d8431073ae03cdaedb198a89722f3aab6d418ef github.com/godbus/dbus 5f6efc7ef2759c81b7ba876593971bfce311eab3 github.com/gogo/protobuf 8d70fb3182befc465c4a1eac8ad4d38ff49778e2 -github.com/golang/protobuf/proto f7137ae6b19afbfd61a94b746fda3b3fe0491874 +github.com/golang/protobuf f7137ae6b19afbfd61a94b746fda3b3fe0491874 github.com/gorilla/context 215affda49addc4c8ef7e2534915df2c8c35c6cd github.com/gorilla/mux 8096f47503459bcc74d1f4c487b7e6e42e5746b5 -github.com/hashicorp/consul/api 954aec66231b79c161a4122b023fbcad13047f79 -github.com/hashicorp/go-msgpack/codec 71c2886f5a673a35f909803f38ece5810165097b +github.com/hashicorp/consul 954aec66231b79c161a4122b023fbcad13047f79 +github.com/hashicorp/go-msgpack 71c2886f5a673a35f909803f38ece5810165097b github.com/hashicorp/go-multierror 2167c8ec40776024589f483a6b836489e47e1049 github.com/hashicorp/memberlist v0.1.0 github.com/sean-/seed e2103e2c35297fb7e17febb81e49b312087a2372 @@ -31,13 +31,13 @@ github.com/hashicorp/go-sockaddr acd314c5781ea706c710d9ea70069fd2e110d61d github.com/hashicorp/serf 598c54895cc5a7b1a24a398d635e8c0ea0959870 github.com/mattn/go-shellwords 525bedee691b5a8df547cb5cf9f86b7fb1883e24 github.com/miekg/dns d27455715200c7d3e321a1e5cadb27c9ee0b0f02 -github.com/opencontainers/runc/libcontainer ba1568de399395774ad84c2ace65937814c542ed -github.com/samuel/go-zookeeper/zk d0e0d8e11f318e000a8cc434616d69e329edc374 +github.com/opencontainers/runc ba1568de399395774ad84c2ace65937814c542ed +github.com/samuel/go-zookeeper d0e0d8e11f318e000a8cc434616d69e329edc374 github.com/seccomp/libseccomp-golang 1b506fc7c24eec5a3693cdcbed40d9c226cfc6a1 github.com/stretchr/testify dab07ac62d4905d3e48d17dc549c684ac3b7c15a -github.com/syndtr/gocapability/capability 2c00daeb6c3b45114c80ac44119e7b8801fdd852 +github.com/syndtr/gocapability 2c00daeb6c3b45114c80ac44119e7b8801fdd852 github.com/ugorji/go f1f1a805ed361a0e078bb537e4ea78cd37dcf065 -github.com/vishvananda/netlink 1e86b2bee5b6a7d377e4c02bb7f98209d6a7297c +github.com/vishvananda/netlink bd6d5de5ccef2d66b0a26177928d0d8895d7f969 github.com/vishvananda/netns 604eaf189ee867d8c147fafc28def2394e878d25 golang.org/x/net c427ad74c6d7a814201695e9ffde0c5d400a7674 golang.org/x/sys 8f0908ab3b2457e2e15403d3697c9ef5cb4b57a9