Skip to content

Commit

Permalink
Add dual stack support
Browse files Browse the repository at this point in the history 
Add new option for flannel daemon to support dual stack:
- "publicIPv6": "IPv6 accessible by other nodes for
  inter-host communication"
- "auto-detect-ipv4": "auto detect ipv4 address of the iface",
  default value is true.
- "auto-detect-ipv6": "auto detect ipv6 address of the iface",
  default value is false

Add new option into `net-conf.json` configuration, like following:
{
  "EnableIPv4": true,
  "EnableIPv6": true,
  "Network": "172.16.0.0/16",
  "IPv6Network": "fc00::/48",
  "Backend": {
    "Type": "vxlan"
  }
}
EnableIPv4 default value is true for useing kube subnet manager.
EnableIpv6 default value is false.

Flannel dual stack feature has limitation, only work with vxlan backend
and kube subnet manager now. To enable flannel dual stack feature, need
to do the following step:
1. setting flanneld daemon with "--kube-subnet-mgr --auto-detect-ipv6"
2. settting "EnableIPv6" and "IPv6Network" in "net-conf.json" like the
above configuration.
3. setting network interface that flannel used ipv6 address and
default ipv6 gateway in the host node.
4. vxlan support ipv6 tunnel require kernel version >= 3.12.

Signed-off-by: yaoice <[email protected]>
  • Loading branch information
@yaoice @manuelbuil
yaoice authored and manuelbuil committed Jul 2, 2021
1 parent 0b3e172 commit c7a7aa8
Show file tree
Hide file tree
Showing 21 changed files with 1,627 additions and 255 deletions.
8 changes: 5 additions & 3 deletions backend/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,11 @@ import (
)

type ExternalInterface struct {
Iface *net.Interface
IfaceAddr net.IP
ExtAddr net.IP
Iface *net.Interface
IfaceAddr net.IP
IfaceV6Addr net.IP
ExtAddr net.IP
ExtV6Addr net.IP
}

// Besides the entry points in the Backend interface, the backend's New()
Expand Down
58 changes: 58 additions & 0 deletions backend/vxlan/device.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,13 +124,26 @@ func (dev *vxlanDevice) Configure(ipa ip.IP4Net, flannelnet ip.IP4Net) error {
return nil
}

func (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net) error {
if err := ip.EnsureV6AddressOnLink(ipn, dev.link); err != nil {
return fmt.Errorf("failed to ensure v6 address of interface %s: %s", dev.link.Attrs().Name, err)
}

if err := netlink.LinkSetUp(dev.link); err != nil {
return fmt.Errorf("failed to set v6 interface %s to UP state: %s", dev.link.Attrs().Name, err)
}

return nil
}

func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
return dev.link.HardwareAddr
}

type neighbor struct {
MAC net.HardwareAddr
IP ip.IP4
IP6 *ip.IP6
}

func (dev *vxlanDevice) AddFDB(n neighbor) error {
Expand All @@ -145,6 +158,18 @@ func (dev *vxlanDevice) AddFDB(n neighbor) error {
})
}

func (dev *vxlanDevice) AddV6FDB(n neighbor) error {
log.V(4).Infof("calling AddV6FDB: %v, %v", n.IP6, n.MAC)
return netlink.NeighSet(&netlink.Neigh{
LinkIndex: dev.link.Index,
State: netlink.NUD_PERMANENT,
Family: syscall.AF_BRIDGE,
Flags: netlink.NTF_SELF,
IP: n.IP6.ToIP(),
HardwareAddr: n.MAC,
})
}

func (dev *vxlanDevice) DelFDB(n neighbor) error {
log.V(4).Infof("calling DelFDB: %v, %v", n.IP, n.MAC)
return netlink.NeighDel(&netlink.Neigh{
Expand All @@ -156,6 +181,17 @@ func (dev *vxlanDevice) DelFDB(n neighbor) error {
})
}

func (dev *vxlanDevice) DelV6FDB(n neighbor) error {
log.V(4).Infof("calling DelV6FDB: %v, %v", n.IP6, n.MAC)
return netlink.NeighDel(&netlink.Neigh{
LinkIndex: dev.link.Index,
Family: syscall.AF_BRIDGE,
Flags: netlink.NTF_SELF,
IP: n.IP6.ToIP(),
HardwareAddr: n.MAC,
})
}

func (dev *vxlanDevice) AddARP(n neighbor) error {
log.V(4).Infof("calling AddARP: %v, %v", n.IP, n.MAC)
return netlink.NeighSet(&netlink.Neigh{
Expand All @@ -167,6 +203,17 @@ func (dev *vxlanDevice) AddARP(n neighbor) error {
})
}

func (dev *vxlanDevice) AddV6ARP(n neighbor) error {
log.V(4).Infof("calling AddV6ARP: %v, %v", n.IP6, n.MAC)
return netlink.NeighSet(&netlink.Neigh{
LinkIndex: dev.link.Index,
State: netlink.NUD_PERMANENT,
Type: syscall.RTN_UNICAST,
IP: n.IP6.ToIP(),
HardwareAddr: n.MAC,
})
}

func (dev *vxlanDevice) DelARP(n neighbor) error {
log.V(4).Infof("calling DelARP: %v, %v", n.IP, n.MAC)
return netlink.NeighDel(&netlink.Neigh{
Expand All @@ -178,6 +225,17 @@ func (dev *vxlanDevice) DelARP(n neighbor) error {
})
}

func (dev *vxlanDevice) DelV6ARP(n neighbor) error {
log.V(4).Infof("calling DelV6ARP: %v, %v", n.IP6, n.MAC)
return netlink.NeighDel(&netlink.Neigh{
LinkIndex: dev.link.Index,
State: netlink.NUD_PERMANENT,
Type: syscall.RTN_UNICAST,
IP: n.IP6.ToIP(),
HardwareAddr: n.MAC,
})
}

func vxlanLinksIncompat(l1, l2 netlink.Link) string {
if l1.Type() != l2.Type() {
return fmt.Sprintf("link type: %v vs %v", l1.Type(), l2.Type())
Expand Down
100 changes: 71 additions & 29 deletions backend/vxlan/vxlan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,19 +88,34 @@ func New(sm subnet.Manager, extIface *backend.ExternalInterface) (backend.Backen
return backend, nil
}

func newSubnetAttrs(publicIP net.IP, vnid uint16, mac net.HardwareAddr) (*subnet.LeaseAttrs, error) {
data, err := json.Marshal(&vxlanLeaseAttrs{
VNI: vnid,
VtepMAC: hardwareAddr(mac)})
if err != nil {
return nil, err
func newSubnetAttrs(publicIP net.IP, publicIPv6 net.IP, vnid uint16, dev, v6Dev *vxlanDevice) (*subnet.LeaseAttrs, error) {
leaseAttrs := &subnet.LeaseAttrs{
BackendType: "vxlan",
}
if publicIP != nil && dev != nil {
data, err := json.Marshal(&vxlanLeaseAttrs{
VNI: vnid,
VtepMAC: hardwareAddr(dev.MACAddr()),
})
if err != nil {
return nil, err
}
leaseAttrs.PublicIP = ip.FromIP(publicIP)
leaseAttrs.BackendData = json.RawMessage(data)
}

return &subnet.LeaseAttrs{
PublicIP: ip.FromIP(publicIP),
BackendType: "vxlan",
BackendData: json.RawMessage(data),
}, nil
if publicIPv6 != nil && v6Dev != nil {
data, err := json.Marshal(&vxlanLeaseAttrs{
VNI: vnid,
VtepMAC: hardwareAddr(v6Dev.MACAddr()),
})
if err != nil {
return nil, err
}
leaseAttrs.PublicIPv6 = ip.FromIP6(publicIPv6)
leaseAttrs.BackendV6Data = json.RawMessage(data)
}
return leaseAttrs, nil
}

func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup, config *subnet.Config) (backend.Network, error) {
Expand All @@ -122,23 +137,43 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
}
log.Infof("VXLAN config: VNI=%d Port=%d GBP=%v Learning=%v DirectRouting=%v", cfg.VNI, cfg.Port, cfg.GBP, cfg.Learning, cfg.DirectRouting)

devAttrs := vxlanDeviceAttrs{
vni: uint32(cfg.VNI),
name: fmt.Sprintf("flannel.%v", cfg.VNI),
vtepIndex: be.extIface.Iface.Index,
vtepAddr: be.extIface.IfaceAddr,
vtepPort: cfg.Port,
gbp: cfg.GBP,
learning: cfg.Learning,
}
var dev, v6Dev *vxlanDevice
var err error
if config.EnableIPv4 {
devAttrs := vxlanDeviceAttrs{
vni: uint32(cfg.VNI),
name: fmt.Sprintf("flannel.%v", cfg.VNI),
vtepIndex: be.extIface.Iface.Index,
vtepAddr: be.extIface.IfaceAddr,
vtepPort: cfg.Port,
gbp: cfg.GBP,
learning: cfg.Learning,
}

dev, err := newVXLANDevice(&devAttrs)
if err != nil {
return nil, err
dev, err = newVXLANDevice(&devAttrs)
if err != nil {
return nil, err
}
dev.directRouting = cfg.DirectRouting
}
if config.EnableIPv6 {
v6DevAttrs := vxlanDeviceAttrs{
vni: uint32(cfg.VNI),
name: fmt.Sprintf("flannel-v6.%v", cfg.VNI),
vtepIndex: be.extIface.Iface.Index,
vtepAddr: be.extIface.IfaceV6Addr,
vtepPort: cfg.Port,
gbp: cfg.GBP,
learning: cfg.Learning,
}
v6Dev, err = newVXLANDevice(&v6DevAttrs)
if err != nil {
return nil, err
}
v6Dev.directRouting = cfg.DirectRouting
}
dev.directRouting = cfg.DirectRouting

subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, uint16(cfg.VNI), dev.MACAddr())
subnetAttrs, err := newSubnetAttrs(be.extIface.ExtAddr, be.extIface.ExtV6Addr, uint16(cfg.VNI), dev, v6Dev)
if err != nil {
return nil, err
}
Expand All @@ -155,11 +190,18 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
// Ensure that the device has a /32 address so that no broadcast routes are created.
// This IP is just used as a source address for host to workload traffic (so
// the return path for the traffic has an address on the flannel network to use as the destination)
if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
return nil, fmt.Errorf("failed to configure interface %s: %s", dev.link.Attrs().Name, err)
if config.EnableIPv4 {
if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
}
}

if config.EnableIPv6 {
if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}); err != nil {
return nil, fmt.Errorf("failed to configure interface %s: %s", v6Dev.link.Attrs().Name, err)
}
}

return newNetwork(be.subnetMgr, be.extIface, dev, ip.IP4Net{}, lease)
return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease)
}

// So we can make it JSON (un)marshalable
Expand Down
Loading

0 comments on commit c7a7aa8

Please sign in to comment.