Add dual-stack support in Canal

Signed-off-by: Manuel Buil <[email protected]>

cni-plugin/internal/pkg/utils/utils.go

@@ -202,16 +202,18 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er
 		// We need to replace "usePodCidr" with a valid, but dummy podCidr string with "host-local" IPAM.
 		// host-local IPAM releases the IP by ContainerID, so podCidr isn't really used to release the IP.
 		// It just needs a valid CIDR, but it doesn't have to be the CIDR associated with the host.
-		const dummyPodCidr = "0.0.0.0/0"
+		dummyPodCidrv4 := "0.0.0.0/0"
+		dummyPodCidrv6 := "::/0"
 		var stdinData map[string]interface{}
 		err := json.Unmarshal(args.StdinData, &stdinData)
 		if err != nil {
 			return err
 		}
 
-		logger.WithField("podCidr", dummyPodCidr).Info("Using a dummy podCidr to release the IP")
-		getDummyPodCIDR := func() (string, error) {
-			return dummyPodCidr, nil
+		logger.WithFields(logrus.Fields{"podCidrv4": dummyPodCidrv4,
+			"podCidrv6": dummyPodCidrv6}).Info("Using dummy podCidrs to release the IPs")
+		getDummyPodCIDR := func() (string, string, error) {
+			return dummyPodCidrv4, dummyPodCidrv6, nil
 		}
 		err = ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getDummyPodCIDR)
 		if err != nil {
@@ -286,13 +288,13 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er
 //      }
 //      ...
 //    }
-func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDR func() (string, error)) error {
+func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDRs func() (string, string, error)) error {
 	ipamData, ok := stdinData["ipam"].(map[string]interface{})
 	if !ok {
 		return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", stdinData["ipam"])
 	}
 	// Older versions of host-local IPAM store a single subnet in the top-level IPAM dict.
-	err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDR)
+	err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDRs)
 	if err != nil {
 		return err
 	}
@@ -310,7 +312,7 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int
 				return fmt.Errorf("failed to parse host-local IPAM range set; was expecting a list, not: %v", rs)
 			}
 			for _, r := range rs {
-				err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDR)
+				err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDRs)
 				if err != nil {
 					return err
 				}
@@ -320,29 +322,47 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int
 	return nil
 }
 
-func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidr func() (string, error)) error {
+func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidrs func() (string, string, error)) error {
 	logrus.WithField("ipamData", rawIpamData).Debug("Examining IPAM data for usePodCidr")
 	ipamData, ok := rawIpamData.(map[string]interface{})
 	if !ok {
 		return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", rawIpamData)
 	}
 	subnet, _ := ipamData["subnet"].(string)
+
 	if strings.EqualFold(subnet, "usePodCidr") {
-		logger.Info("Calico CNI fetching podCidr from Kubernetes")
-		podCidr, err := getPodCidr()
+		ipv4Cidr, _, err := getPodCidrs()
+		if err != nil {
+			logger.Errorf("Failed to getPodCidrs")
+			return err
+		}
+		if ipv4Cidr == "" {
+			return errors.New("usePodCidr found but there is no IPv4 CIDR configured")
+		}
+
+		ipamData["subnet"] = ipv4Cidr
+		subnet = ipv4Cidr
+		logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", ipv4Cidr)
+
+		// updateHostLocalIPAMDataForOS is only required for Windows and only ipv4 is supported
+		err = updateHostLocalIPAMDataForOS(subnet, ipamData)
 		if err != nil {
-			logger.Info("Failed to getPodCidr")
 			return err
 		}
-		logger.WithField("podCidr", podCidr).Info("Fetched podCidr")
-		ipamData["subnet"] = podCidr
-		subnet = podCidr
-		logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", podCidr)
 	}
 
-	err := updateHostLocalIPAMDataForOS(subnet, ipamData)
-	if err != nil {
-		return err
+	if strings.EqualFold(subnet, "usePodCidrIPv6") {
+		_, ipv6Cidr, err := getPodCidrs()
+		if err != nil {
+			logger.Errorf("Failed to ipv6 getPodCidrs")
+			return err
+		}
+		if ipv6Cidr == "" {
+			return errors.New("usePodCidrIPv6 found but there is no IPv6 CIDR configured")
+		}
+
+		ipamData["subnet"] = ipv6Cidr
+		logger.Infof("Calico CNI passing podCidrv6 to host-local IPAM: %s", ipv6Cidr)
 	}
 
 	return nil
@@ -360,6 +380,7 @@ func UpdateHostLocalIPAMDataForWindows(subnet string, ipamData map[string]interf
 		return err
 	}
 	//process only if we have ipv4 subnet
+	//VXLAN networks on Windows do not support dual-stack https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv6-networking
 	if ip.To4() != nil {
 		//get Expected start and end range for given CIDR
 		expStartRange, expEndRange := getIPRanges(ip, ipnet)

cni-plugin/pkg/k8s/k8s.go

@@ -105,18 +105,24 @@ func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, epID
 		}
 
 		// Defer to ReplaceHostLocalIPAMPodCIDRs to swap the "usePodCidr" value out.
-		var cachedPodCidr string
-		getRealPodCIDR := func() (string, error) {
-			if cachedPodCidr == "" {
+		var cachedPodCidrs []string
+		var cachedIpv4Cidr, cachedIpv6Cidr string
+		getRealPodCIDRs := func() (string, string, error) {
+			if len(cachedPodCidrs) == 0 {
 				var err error
-				cachedPodCidr, err = getPodCidr(client, conf, epIDs.Node)
+				var emptyResult string
+				cachedPodCidrs, err = getPodCidrs(client, conf, epIDs.Node)
 				if err != nil {
-					return "", err
+					return emptyResult, emptyResult, err
+				}
+				cachedIpv4Cidr, cachedIpv6Cidr, err = getIPsByFamily(cachedPodCidrs)
+				if err != nil {
+					return emptyResult, emptyResult, err
 				}
 			}
-			return cachedPodCidr, nil
+			return cachedIpv4Cidr, cachedIpv6Cidr, nil
 		}
-		err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDR)
+		err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDRs)
 		if err != nil {
 			return nil, err
 		}
@@ -891,19 +897,44 @@ func getK8sPodInfo(client *kubernetes.Clientset, podName, podNamespace string) (
 	return labels, pod.Annotations, ports, profiles, generateName, serviceAccount, nil
 }
 
-func getPodCidr(client *kubernetes.Clientset, conf types.NetConf, nodename string) (string, error) {
+// getPodCidrs returns the podCidrs included in the node manifest
+func getPodCidrs(client *kubernetes.Clientset, conf types.NetConf, nodename string) ([]string, error) {
+	var emptyString []string
 	// Pull the node name out of the config if it's set. Defaults to nodename
 	if conf.Kubernetes.NodeName != "" {
 		nodename = conf.Kubernetes.NodeName
 	}
 
 	node, err := client.CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{})
 	if err != nil {
-		return "", err
+		return emptyString, err
+	}
+	if len(node.Spec.PodCIDRs) == 0 {
+		return emptyString, fmt.Errorf("no podCidr for node %s", nodename)
+	}
+	return node.Spec.PodCIDRs, nil
+}
+
+// getIPsByFamily returns the IPv4 and IPv6 CIDRs
+func getIPsByFamily(cidrs []string) (string, string, error) {
+	var ipv4Cidr, ipv6Cidr string
+	for _, cidr := range cidrs {
+		_, ipNet, err := cnet.ParseCIDR(cidr)
+		if err != nil {
+			return "", "", err
+		}
+		if ipNet.Version() == 4 {
+			ipv4Cidr = cidr
+		}
+
+		if ipNet.Version() == 6 {
+			ipv6Cidr = cidr
+		}
 	}
 
-	if node.Spec.PodCIDR == "" {
-		return "", fmt.Errorf("no podCidr for node %s", nodename)
+	if (len(cidrs) > 1) && (ipv4Cidr == "" || ipv6Cidr == "") {
+		return "", "", errors.New("ClusterCIDR contains two ranges of the same type")
 	}
-	return node.Spec.PodCIDR, nil
+
+	return ipv4Cidr, ipv6Cidr, nil
 }

cni-plugin/tests/calico_cni_k8s_test.go

@@ -708,6 +708,52 @@ var _ = Describe("Kubernetes CNI tests", func() {
 				numIPv4IPs:      1,
 				numIPv6IPs:      1,
 			},
+			{
+				// This scenario tests IPv4+IPv6 without specifying any routes.
+				description: "new-style with IPv4 and IPv6 both using usePodCidr, no routes",
+				cniVersion:  "0.3.0",
+				config: `
+					{
+					  "cniVersion": "%s",
+					  "name": "net6",
+					  "nodename_file_optional": true,
+					  "type": "calico",
+					  "etcd_endpoints": "http://%s:2379",
+					  "datastore_type": "%s",
+					  "ipam": {
+					    "type": "host-local",
+					    "ranges": [
+					       [
+					         {
+					           "subnet": "usePodCidr"
+					         }
+					       ],
+					       [
+					         {
+					           "subnet": "usePodCidrIPv6"
+					         }
+					       ]
+					    ]
+					  },
+					  "kubernetes": {
+                                           "kubeconfig": "/home/user/certs/kubeconfig"
+					  },
+					  "policy": {"type": "k8s"},
+					  "log_level":"info"
+					}`,
+				expectedV4Routes: []string{
+					regexp.QuoteMeta("default via 169.254.1.1 dev eth0"),
+					regexp.QuoteMeta("169.254.1.1 dev eth0 scope link"),
+				},
+				expectedV6Routes: []string{
+					"dead:beef::[0-9a-f]* dev eth0 proto kernel metric 256 pref medium",
+					"fe80::/64 dev eth0 proto kernel metric 256 pref medium",
+					"default via fe80::ecee:eeff:feee:eeee dev eth0 metric 1024",
+				},
+				unexpectedRoute: regexp.QuoteMeta("10."),
+				numIPv4IPs:      1,
+				numIPv6IPs:      1,
+			},
 			{
 				// In this scenario, we use a lot more of the host-local IPAM plugin.  Namely:
 				// - we use multiple ranges, one of which is IPv6, the other uses the podCIDR
@@ -831,8 +877,13 @@ var _ = Describe("Kubernetes CNI tests", func() {
 			},
 		}
 
+		// Run tests with PodCIDR
 		for _, c := range hostLocalIPAMConfigs {
 			c := c // Make sure we get a fresh variable on each loop.
+			// The dual-stack requires PodCIDRs
+			if strings.Contains(c.config, "usePodCidrIPv6") {
+				continue
+			}
 			Context("Using host-local IPAM ("+c.description+"): request an IP then release it, and then request it again", func() {
 				It("should successfully assign IP both times and successfully release it in the middle", func() {
 					netconfHostLocalIPAM := fmt.Sprintf(c.config, c.cniVersion, os.Getenv("ETCD_IP"), os.Getenv("DATASTORE_TYPE"))
@@ -942,6 +993,117 @@ var _ = Describe("Kubernetes CNI tests", func() {
 			})
 		}
 
+		// Run tests with PodCIDRs defining a dual-stack deployment
+		for _, c := range hostLocalIPAMConfigs {
+			c := c // Make sure we get a fresh variable on each loop.
+			Context("Using host-local IPAM ("+c.description+"): request an IP then release it, and then request it again", func() {
+				It("should successfully assign IP both times and successfully release it in the middle", func() {
+					netconfHostLocalIPAM := fmt.Sprintf(c.config, c.cniVersion, os.Getenv("ETCD_IP"), os.Getenv("DATASTORE_TYPE"))
+
+					clientset := getKubernetesClient()
+
+					ensureNamespace(clientset, testutils.K8S_TEST_NS)
+
+					ensureNodeDeleted(clientset, hostname)
+
+					// Create a K8s Node object with PodCIDR and name equal to hostname.
+					_, err = clientset.CoreV1().Nodes().Create(context.Background(), &v1.Node{
+						ObjectMeta: metav1.ObjectMeta{Name: hostname},
+						Spec: v1.NodeSpec{
+							PodCIDRs: []string{"10.10.0.0/24", "dead:beef::/96"},
+						},
+					}, metav1.CreateOptions{})
+					Expect(err).NotTo(HaveOccurred())
+					defer ensureNodeDeleted(clientset, hostname)
+
+					By("Creating a pod with a specific IP address")
+					name := fmt.Sprintf("run%d", rand.Uint32())
+					ensurePodCreated(clientset, testutils.K8S_TEST_NS, &v1.Pod{
+						ObjectMeta: metav1.ObjectMeta{Name: name},
+						Spec: v1.PodSpec{
+							Containers: []v1.Container{{
+								Name:  name,
+								Image: "ignore",
+							}},
+							NodeName: hostname,
+						},
+					})
+					defer ensurePodDeleted(clientset, testutils.K8S_TEST_NS, name)
+
+					requestedIP := "10.10.0.42"
+					expectedIP := net.IPv4(10, 10, 0, 42).To4()
+
+					_, _, _, contAddresses, _, contNs, err := testutils.CreateContainer(netconfHostLocalIPAM, name, testutils.K8S_TEST_NS, requestedIP)
+					Expect(err).NotTo(HaveOccurred())
+
+					podIP := contAddresses[0].IP
+					Expect(podIP).Should(Equal(expectedIP))
+
+					By("Deleting the pod we created earlier")
+					_, err = testutils.DeleteContainer(netconfHostLocalIPAM, contNs.Path(), name, testutils.K8S_TEST_NS)
+					Expect(err).ShouldNot(HaveOccurred())
+
+					By("Creating a second pod with the same IP address as the first pod")
+					name2 := fmt.Sprintf("run2%d", rand.Uint32())
+					ensurePodCreated(clientset, testutils.K8S_TEST_NS, &v1.Pod{
+						ObjectMeta: metav1.ObjectMeta{Name: name2},
+						Spec: v1.PodSpec{
+							Containers: []v1.Container{{
+								Name:  fmt.Sprintf("container-%s", name2),
+								Image: "ignore",
+							}},
+							NodeName: hostname,
+						},
+					})
+					defer ensurePodDeleted(clientset, testutils.K8S_TEST_NS, name2)
+
+					_, _, _, contAddresses, _, contNs, err = testutils.CreateContainer(netconfHostLocalIPAM, name2, testutils.K8S_TEST_NS, requestedIP)
+					Expect(err).NotTo(HaveOccurred())
+
+					pod2IP := contAddresses[0].IP
+					Expect(pod2IP).Should(Equal(expectedIP))
+
+					err = contNs.Do(func(_ ns.NetNS) error {
+						defer GinkgoRecover()
+						out, err := exec.Command("ip", "route", "show").Output()
+						Expect(err).NotTo(HaveOccurred())
+						for _, r := range c.expectedV4Routes {
+							Expect(string(out)).To(MatchRegexp(r))
+						}
+
+						if c.unexpectedRoute != "" {
+							Expect(string(out)).NotTo(ContainSubstring(c.unexpectedRoute))
+						}
+
+						out, err = exec.Command("ip", "-6", "route", "show").Output()
+						Expect(err).NotTo(HaveOccurred())
+						for _, r := range c.expectedV6Routes {
+							Expect(string(out)).To(MatchRegexp(r))
+						}
+
+						if c.numIPv6IPs > 0 {
+							err := testutils.CheckSysctlValue("/proc/sys/net/ipv6/conf/eth0/accept_dad", "0")
+							Expect(err).NotTo(HaveOccurred())
+						}
+
+						out, err = exec.Command("ip", "addr", "show").Output()
+						Expect(err).NotTo(HaveOccurred())
+						inet := regexp.MustCompile(` {4}inet .*scope global`)
+						Expect(inet.FindAll(out, -1)).To(HaveLen(c.numIPv4IPs))
+						inetv6 := regexp.MustCompile(` {4}inet6 .*scope global`)
+						Expect(inetv6.FindAll(out, -1)).To(HaveLen(c.numIPv6IPs))
+						Expect(out).NotTo(ContainSubstring("scope global tentative"),
+							"Some IPv6 addresses marked as tentative; disabling DAD must have failed.")
+
+						return nil
+					})
+					Expect(err).ShouldNot(HaveOccurred())
+
+					_, err = testutils.DeleteContainer(netconfHostLocalIPAM, contNs.Path(), name2, testutils.K8S_TEST_NS)
+					Expect(err).ShouldNot(HaveOccurred())
+				})
+			})
+		}
 	})
 
 	Context("using calico-ipam with a Namespace annotation only", func() {