Changes on top of dualStack PR

This patch addresses the reviews made on the original PR

Signed-off-by: Manuel Buil <[email protected]>

Documentation/configuration.md

@@ -80,3 +80,26 @@ Any command line option can be turned into an environment variable by prefixing
 Flannel provides a health check http endpoint `healthz`. Currently this endpoint will blindly
 return http status ok(i.e. 200) when flannel is running. This feature is by default disabled.
 Set `healthz-port` to a non-zero value will enable a healthz server for flannel.
+
+## Dual-stack
+
+Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan backend.
+
+Requirements:
+* v1.0 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins)
+* Nodes must have an ipv4 and ipv6 address in the main interface
+* Nodes must have an ipv4 and ipv6 address default route
+* vxlan support ipv6 tunnel require kernel version >= 3.12
+
+Configuration:
+* Set flanneld daemon with "--kube-subnet-mgr" CLI option
+* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap 
+
+If everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example:
+
+FLANNEL_NETWORK=10.42.0.0/16
+FLANNEL_SUBNET=10.42.0.1/24
+FLANNEL_IPV6_NETWORK=2001:cafe:42::/56
+FLANNEL_IPV6_SUBNET=2001:cafe:42::1/64
+FLANNEL_MTU=1450
+FLANNEL_IPMASQ=true

backend/vxlan/device.go

@@ -124,13 +124,13 @@ func (dev *vxlanDevice) Configure(ipa ip.IP4Net, flannelnet ip.IP4Net) error {
 	return nil
 }
 
-func (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net) error {
-	if err := ip.EnsureV6AddressOnLink(ipn, dev.link); err != nil {
-		return fmt.Errorf("failed to ensure v6 address of interface %s: %s", dev.link.Attrs().Name, err)
+func (dev *vxlanDevice) ConfigureIPv6(ipn ip.IP6Net, flannelnet ip.IP6Net) error {
+	if err := ip.EnsureV6AddressOnLink(ipn, flannelnet, dev.link); err != nil {
+		return fmt.Errorf("failed to ensure v6 address of interface %s: %w", dev.link.Attrs().Name, err)
 	}
 
 	if err := netlink.LinkSetUp(dev.link); err != nil {
-		return fmt.Errorf("failed to set v6 interface %s to UP state: %s", dev.link.Attrs().Name, err)
+		return fmt.Errorf("failed to set v6 interface %s to UP state: %w", dev.link.Attrs().Name, err)
 	}
 
 	return nil
@@ -142,7 +142,7 @@ func (dev *vxlanDevice) MACAddr() net.HardwareAddr {
 
 type neighbor struct {
 	MAC net.HardwareAddr
-	IP  ip.IP4
+	IP ip.IP4
 	IP6 *ip.IP6
 }
 

backend/vxlan/vxlan.go

@@ -190,15 +190,14 @@ func (be *VXLANBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup,
 	// Ensure that the device has a /32 address so that no broadcast routes are created.
 	// This IP is just used as a source address for host to workload traffic (so
 	// the return path for the traffic has an address on the flannel network to use as the destination)
-        if config.EnableIPv4 {
-                if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
-                        return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
-                }
-        }
-
+	if config.EnableIPv4 {
+		if err := dev.Configure(ip.IP4Net{IP: lease.Subnet.IP, PrefixLen: 32}, config.Network); err != nil {
+			return nil, fmt.Errorf("failed to configure interface %s: %w", dev.link.Attrs().Name, err)
+		}
+	}
 	if config.EnableIPv6 {
-		if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}); err != nil {
-			return nil, fmt.Errorf("failed to configure interface %s: %s", v6Dev.link.Attrs().Name, err)
+		if err := v6Dev.ConfigureIPv6(ip.IP6Net{IP: lease.IPv6Subnet.IP, PrefixLen: 128}, config.IPv6Network); err != nil {
+			return nil, fmt.Errorf("failed to configure interface %s: %w", v6Dev.link.Attrs().Name, err)
 		}
 	}
 	return newNetwork(be.subnetMgr, be.extIface, dev, v6Dev, ip.IP4Net{}, lease)

backend/vxlan/vxlan_network.go

@@ -96,7 +96,7 @@ func (nw *network) handleSubnetEvents(batch []subnet.Event) {
 		v6Sn := event.Lease.IPv6Subnet
 		attrs := event.Lease.Attrs
 		if attrs.BackendType != "vxlan" {
-			log.Warningf("ignoring non-vxlan v4Subnet(%s) v6Subnet: type=%v", sn, v6Sn, attrs.BackendType)
+			log.Warningf("ignoring non-vxlan v4Subnet(%s) v6Subnet(%s): type=%v", sn, v6Sn, attrs.BackendType)
 			continue
 		}
 

go.sum

@@ -73,6 +73,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=

main.go

@@ -134,8 +134,6 @@ func init() {
 	flannelFlags.StringVar(&opts.publicIPv6, "public-ipv6", "", "IPv6 accessible by other nodes for inter-host communication")
 	flannelFlags.IntVar(&opts.subnetLeaseRenewMargin, "subnet-lease-renew-margin", 60, "subnet lease renewal margin, in minutes, ranging from 1 to 1439")
 	flannelFlags.BoolVar(&opts.ipMasq, "ip-masq", false, "setup IP masquerade rule for traffic destined outside of overlay network")
-	flannelFlags.BoolVar(&opts.autoDetectIPv4, "auto-detect-ipv4", true, "auto detect ipv4 address of the iface")
-	flannelFlags.BoolVar(&opts.autoDetectIPv6, "auto-detect-ipv6", false, "auto detect ipv6 address of the iface")
 	flannelFlags.BoolVar(&opts.kubeSubnetMgr, "kube-subnet-mgr", false, "contact the Kubernetes API for subnet assignment instead of etcd.")
 	flannelFlags.StringVar(&opts.kubeApiUrl, "kube-api-url", "", "Kubernetes API server URL. Does not need to be specified if flannel is running in a pod.")
 	flannelFlags.StringVar(&opts.kubeAnnotationPrefix, "kube-annotation-prefix", "flannel.alpha.coreos.com", `Kubernetes annotation prefix. Can contain single slash "/", otherwise it will be appended at the end.`)
@@ -225,15 +223,53 @@ func main() {
 		os.Exit(1)
 	}
 
+	// This is the main context that everything should run in.
+	// All spawned goroutines should exit when cancel is called on this context.
+	// Go routines spawned from main.go coordinate using a WaitGroup. This provides a mechanism to allow the shutdownHandler goroutine
+	// to block until all the goroutines return . If those goroutines spawn other goroutines then they are responsible for
+	// blocking and returning only when cancel() is called.
+	ctx, cancel := context.WithCancel(context.Background())
+
+	sm, err := newSubnetManager(ctx)
+	if err != nil {
+		log.Error("Failed to create SubnetManager: ", err)
+		os.Exit(1)
+	}
+	log.Infof("Created subnet manager: %s", sm.Name())
+
+	// Register for SIGINT and SIGTERM
+	log.Info("Installing signal handlers")
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
+
+	wg := sync.WaitGroup{}
+
+	wg.Add(1)
+	go func() {
+		shutdownHandler(ctx, sigs, cancel)
+		wg.Done()
+	}()
+
+	if opts.healthzPort > 0 {
+		// It's not super easy to shutdown the HTTP server so don't attempt to stop it cleanly
+		go mustRunHealthz()
+	}
+
+	// Fetch the network config (i.e. what backend to use etc..).
+	config, err := getConfig(ctx, sm)
+	if err == errCanceled {
+		wg.Wait()
+		os.Exit(0)
+	}
+
 	// Get ip family stack
-	ipStack, stackErr := getIPFamily(opts.autoDetectIPv4, opts.autoDetectIPv6)
+	ipStack, stackErr := getIPFamily(config.EnableIPv4, config.EnableIPv6)
 	if stackErr != nil {
 		log.Error(stackErr.Error())
 		os.Exit(1)
 	}
 	// Work out which interface to use
 	var extIface *backend.ExternalInterface
-	var err error
 	// Check the default interface only if no interfaces are specified
 	if len(opts.iface) == 0 && len(opts.ifaceRegex) == 0 {
 		extIface, err = LookupExtIface(opts.publicIP, "", ipStack)
@@ -275,44 +311,7 @@ func main() {
 		}
 	}
 
-	// This is the main context that everything should run in.
-	// All spawned goroutines should exit when cancel is called on this context.
-	// Go routines spawned from main.go coordinate using a WaitGroup. This provides a mechanism to allow the shutdownHandler goroutine
-	// to block until all the goroutines return . If those goroutines spawn other goroutines then they are responsible for
-	// blocking and returning only when cancel() is called.
-	ctx, cancel := context.WithCancel(context.Background())
-
-	sm, err := newSubnetManager(ctx)
-	if err != nil {
-		log.Error("Failed to create SubnetManager: ", err)
-		os.Exit(1)
-	}
-	log.Infof("Created subnet manager: %s", sm.Name())
-
-	// Register for SIGINT and SIGTERM
-	log.Info("Installing signal handlers")
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
-
-	wg := sync.WaitGroup{}
 
-	wg.Add(1)
-	go func() {
-		shutdownHandler(ctx, sigs, cancel)
-		wg.Done()
-	}()
-
-	if opts.healthzPort > 0 {
-		// It's not super easy to shutdown the HTTP server so don't attempt to stop it cleanly
-		go mustRunHealthz()
-	}
-
-	// Fetch the network config (i.e. what backend to use etc..).
-	config, err := getConfig(ctx, sm)
-	if err == errCanceled {
-		wg.Wait()
-		os.Exit(0)
-	}
 
 	// Create a backend manager then use it to create the backend and register the network with it.
 	bm := backend.NewManager(ctx, sm, extIface)
@@ -517,12 +516,17 @@ func MonitorLease(ctx context.Context, sm subnet.Manager, bn backend.Network, wg
 	}
 }
 
-func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.ExternalInterface, error) {
+func LookupExtIface(ifname string, ifregexS string, ipStack int) (*backend.ExternalInterface, error) {
 	var iface *net.Interface
 	var ifaceAddr net.IP
 	var ifaceV6Addr net.IP
 	var err error
 
+        ifregex, err := regexp.Compile(ifregexS)
+	if err != nil {
+		return nil, fmt.Errorf("could not compile the IP address regex '%s': %w", ifregexS, err)
+	}
+
 	// Check ip family stack
 	if ipStack == noneStack {
 		return nil, fmt.Errorf("none matched ip stack")
@@ -578,9 +582,9 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 					continue
 				}
 
-				matched, err := regexp.MatchString(ifregex, ifaceIP.String())
+				matched, err := ifregex.MatchString(ifaceIP.String())
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
 				}
 
 				if matched {
@@ -595,9 +599,9 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 					continue
 				}
 
-				matched, err := regexp.MatchString(ifregex, ifaceIP.String())
+				matched, err := ifregex.MatchString(ifaceIP.String())
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
 				}
 
 				if matched {
@@ -612,9 +616,9 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 					continue
 				}
 
-				matched, err := regexp.MatchString(ifregex, ifaceIP.String())
+				matched, err := ifregex.MatchString(ifaceIP.String())
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
 				}
 
 				ifaceV6IP, err := ip.GetInterfaceIP6Addr(&ifaceToMatch)
@@ -623,9 +627,9 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 					continue
 				}
 
-				v6Matched, err := regexp.MatchString(ifregex, ifaceV6IP.String())
+				v6Matched, err := ifregex.MatchString(ifaceV6IP.String())
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceIP.String())
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceIP.String())
 				}
 
 				if matched && v6Matched {
@@ -640,9 +644,9 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 		// Check Name
 		if iface == nil && (ifaceAddr == nil || ifaceV6Addr == nil) {
 			for _, ifaceToMatch := range ifaces {
-				matched, err := regexp.MatchString(ifregex, ifaceToMatch.Name)
+				matched, err := ifregex.MatchString(ifaceToMatch.Name)
 				if err != nil {
-					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregex, ifaceToMatch.Name)
+					return nil, fmt.Errorf("regex error matching pattern %s to %s", ifregexS, ifaceToMatch.Name)
 				}
 
 				if matched {
@@ -666,26 +670,26 @@ func LookupExtIface(ifname string, ifregex string, ipStack int) (*backend.Extern
 				availableFaces = append(availableFaces, fmt.Sprintf("%s:%s", f.Name, ipaddr))
 			}
 
-			return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregex, strings.Join(availableFaces, ", "))
+			return nil, fmt.Errorf("Could not match pattern %s to any of the available network interfaces (%s)", ifregexS, strings.Join(availableFaces, ", "))
 		}
 	} else {
 		log.Info("Determining IP address of default interface")
 		switch ipStack {
 		case ipv4Stack:
 			if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
-				return nil, fmt.Errorf("failed to get default interface: %s", err)
+				return nil, fmt.Errorf("failed to get default interface: %w", err)
 			}
 		case ipv6Stack:
 			if iface, err = ip.GetDefaultV6GatewayInterface(); err != nil {
-				return nil, fmt.Errorf("failed to get default v6 interface: %s", err)
+				return nil, fmt.Errorf("failed to get default v6 interface: %w", err)
 			}
 		case dualStack:
 			if iface, err = ip.GetDefaultGatewayInterface(); err != nil {
-				return nil, fmt.Errorf("failed to get default interface: %s", err)
+				return nil, fmt.Errorf("failed to get default interface: %w", err)
 			}
 			v6Iface, err := ip.GetDefaultV6GatewayInterface()
 			if err != nil {
-				return nil, fmt.Errorf("failed to get default v6 interface: %s", err)
+				return nil, fmt.Errorf("failed to get default v6 interface: %w", err)
 			}
 			if iface.Name != v6Iface.Name {
 				return nil, fmt.Errorf("v6 default route interface %s "+

pkg/ip/iface.go

@@ -261,24 +261,19 @@ func EnsureV4AddressOnLink(ipa IP4Net, ipn IP4Net, link netlink.Link) error {
 
 // EnsureV6AddressOnLink ensures that there is only one v6 Addr on `link` and it equals `ipn`.
 // If there exist multiple addresses on link, it returns an error message to tell callers to remove additional address.
-func EnsureV6AddressOnLink(ipn IP6Net, link netlink.Link) error {
-	addr := netlink.Addr{IPNet: ipn.ToIPNet()}
+func EnsureV6AddressOnLink(ipa IP6Net, ipn IP6Net, link netlink.Link) error {
+	addr := netlink.Addr{IPNet: ipa.ToIPNet()}
 	existingAddrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
 	if err != nil {
 		return err
 	}
 
-	// flannel will never make this happen. This situation can only be caused by a user, so get them to sort it out.
-	if len(existingAddrs) > 2 {
-		return fmt.Errorf("link has incompatible v6 addresses. Remove additional v6 addresses and try again. %#v", link)
-	}
-
 	onlyLinkLocal := true
 	for _, existingAddr := range existingAddrs {
 		if !existingAddr.IP.IsLinkLocalUnicast() {
 			if !existingAddr.Equal(addr) {
 				if err := netlink.AddrDel(link, &existingAddr); err != nil {
-					return fmt.Errorf("failed to remove v6 IP address %s from %s: %s", ipn.String(), link.Attrs().Name, err)
+					return fmt.Errorf("failed to remove v6 IP address %s from %s: %w", ipn.String(), link.Attrs().Name, err)
 				}
 				existingAddrs = []netlink.Addr{}
 				onlyLinkLocal = false
@@ -295,7 +290,7 @@ func EnsureV6AddressOnLink(ipn IP6Net, link netlink.Link) error {
 	// Actually add the desired address to the interface if needed.
 	if len(existingAddrs) == 0 {
 		if err := netlink.AddrAdd(link, &addr); err != nil {
-			return fmt.Errorf("failed to add v6 IP address %s to %s: %s", ipn.String(), link.Attrs().Name, err)
+			return fmt.Errorf("failed to add v6 IP address %s to %s: %w", ipn.String(), link.Attrs().Name, err)
 		}
 	}
 

pkg/ip/iface_test.go

@@ -74,7 +74,8 @@ func TestEnsureV6AddressOnLink(t *testing.T) {
 		t.Fatal(err)
 	}
 	// check changing address
-	if err := EnsureV6AddressOnLink(IP6Net{IP: FromIP6(net.ParseIP("::2")), PrefixLen: 64}, lo); err != nil {
+	ipn := IP6Net{IP: FromIP6(net.ParseIP("::2")), PrefixLen: 64}
+	if err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {
 		t.Fatal(err)
 	}
 	addrs, err := netlink.AddrList(lo, netlink.FAMILY_V6)
@@ -86,13 +87,21 @@ func TestEnsureV6AddressOnLink(t *testing.T) {
 	}
 
 	// check changing address if there exist multiple addresses
-	if err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("::3"), Mask: net.CIDRMask(64, 128)}}); err != nil {
+	if err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("2001::4"), Mask: net.CIDRMask(64, 128)}}); err != nil {
 		t.Fatal(err)
 	}
-	if err := netlink.AddrAdd(lo, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("::4"), Mask: net.CIDRMask(64, 128)}}); err != nil {
+	addrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)
+	if len(addrs) != 2 {
+		t.Fatalf("two addresses expected, addrs: %v", addrs)
+	}
+	if err := EnsureV6AddressOnLink(ipn, ipn, lo); err != nil {
+		t.Fatal(err)
+	}
+	addrs, err = netlink.AddrList(lo, netlink.FAMILY_V6)
+	if err != nil {
 		t.Fatal(err)
 	}
-	if err := EnsureV6AddressOnLink(IP6Net{IP: FromIP6(net.ParseIP("::2")), PrefixLen: 64}, lo); err == nil {
-		t.Fatal("EnsureV6AddressOnLink should return error if there exist thress address on link")
+	if len(addrs) != 1 {
+		t.Fatalf("only one address expected, addrs: %v", addrs)
 	}
 }

subnet/config.go

@@ -60,9 +60,6 @@ func ParseConfig(s string) (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	if !cfg.EnableIPv4 && !cfg.EnableIPv6 {
-		return nil, fmt.Errorf("EnableIPv4 or EnableIPv6 option must be enabled one at least")
-	}
 
 	if cfg.EnableIPv4 {
 		if cfg.SubnetLen > 0 {