From 32d008ea399e93cd869d4cc5a82d96031c84cb30 Mon Sep 17 00:00:00 2001 From: Manuel Buil Date: Thu, 16 Dec 2021 14:01:44 +0100 Subject: [PATCH] Add dual-stack support in Canal Signed-off-by: Manuel Buil --- cni-plugin/internal/pkg/utils/utils.go | 57 ++++++--- cni-plugin/pkg/k8s/k8s.go | 55 ++++++-- cni-plugin/tests/calico_cni_k8s_test.go | 162 ++++++++++++++++++++++++ 3 files changed, 244 insertions(+), 30 deletions(-) diff --git a/cni-plugin/internal/pkg/utils/utils.go b/cni-plugin/internal/pkg/utils/utils.go index 8e7dc785e53..ff7a30a633a 100644 --- a/cni-plugin/internal/pkg/utils/utils.go +++ b/cni-plugin/internal/pkg/utils/utils.go @@ -202,16 +202,18 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er // We need to replace "usePodCidr" with a valid, but dummy podCidr string with "host-local" IPAM. // host-local IPAM releases the IP by ContainerID, so podCidr isn't really used to release the IP. // It just needs a valid CIDR, but it doesn't have to be the CIDR associated with the host. - const dummyPodCidr = "0.0.0.0/0" + dummyPodCidrv4 := "0.0.0.0/0" + dummyPodCidrv6 := "::/0" var stdinData map[string]interface{} err := json.Unmarshal(args.StdinData, &stdinData) if err != nil { return err } - logger.WithField("podCidr", dummyPodCidr).Info("Using a dummy podCidr to release the IP") - getDummyPodCIDR := func() (string, error) { - return dummyPodCidr, nil + logger.WithFields(logrus.Fields{"podCidrv4": dummyPodCidrv4, + "podCidrv6": dummyPodCidrv6}).Info("Using dummy podCidrs to release the IPs") + getDummyPodCIDR := func() (string, string, error) { + return dummyPodCidrv4, dummyPodCidrv6, nil } err = ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getDummyPodCIDR) if err != nil { @@ -286,13 +288,13 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er // } // ... // } -func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDR func() (string, error)) error { +func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDRs func() (string, string, error)) error { ipamData, ok := stdinData["ipam"].(map[string]interface{}) if !ok { return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", stdinData["ipam"]) } // Older versions of host-local IPAM store a single subnet in the top-level IPAM dict. - err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDR) + err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDRs) if err != nil { return err } @@ -310,7 +312,7 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int return fmt.Errorf("failed to parse host-local IPAM range set; was expecting a list, not: %v", rs) } for _, r := range rs { - err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDR) + err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDRs) if err != nil { return err } @@ -320,29 +322,47 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int return nil } -func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidr func() (string, error)) error { +func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidrs func() (string, string, error)) error { logrus.WithField("ipamData", rawIpamData).Debug("Examining IPAM data for usePodCidr") ipamData, ok := rawIpamData.(map[string]interface{}) if !ok { return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", rawIpamData) } subnet, _ := ipamData["subnet"].(string) + if strings.EqualFold(subnet, "usePodCidr") { - logger.Info("Calico CNI fetching podCidr from Kubernetes") - podCidr, err := getPodCidr() + ipv4Cidr, _, err := getPodCidrs() + if err != nil { + logger.Errorf("Failed to getPodCidrs") + return err + } + if ipv4Cidr == "" { + return errors.New("usePodCidr found but there is no IPv4 CIDR configured") + } + + ipamData["subnet"] = ipv4Cidr + subnet = ipv4Cidr + logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", ipv4Cidr) + + // updateHostLocalIPAMDataForOS is only required for Windows and only ipv4 is supported + err = updateHostLocalIPAMDataForOS(subnet, ipamData) if err != nil { - logger.Info("Failed to getPodCidr") return err } - logger.WithField("podCidr", podCidr).Info("Fetched podCidr") - ipamData["subnet"] = podCidr - subnet = podCidr - logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", podCidr) } - err := updateHostLocalIPAMDataForOS(subnet, ipamData) - if err != nil { - return err + if strings.EqualFold(subnet, "usePodCidrIPv6") { + _, ipv6Cidr, err := getPodCidrs() + if err != nil { + logger.Errorf("Failed to ipv6 getPodCidrs") + return err + } + if ipv6Cidr == "" { + return errors.New("usePodCidrIPv6 found but there is no IPv6 CIDR configured") + } + + ipamData["subnet"] = ipv6Cidr + logger.Infof("Calico CNI passing podCidrv6 to host-local IPAM: %s", ipv6Cidr) } return nil @@ -360,6 +380,7 @@ func UpdateHostLocalIPAMDataForWindows(subnet string, ipamData map[string]interf return err } //process only if we have ipv4 subnet + //VXLAN networks on Windows do not support dual-stack https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv6-networking if ip.To4() != nil { //get Expected start and end range for given CIDR expStartRange, expEndRange := getIPRanges(ip, ipnet) diff --git a/cni-plugin/pkg/k8s/k8s.go b/cni-plugin/pkg/k8s/k8s.go index 866e7f6b7c4..05431977f05 100644 --- a/cni-plugin/pkg/k8s/k8s.go +++ b/cni-plugin/pkg/k8s/k8s.go @@ -105,18 +105,24 @@ func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, epID } // Defer to ReplaceHostLocalIPAMPodCIDRs to swap the "usePodCidr" value out. - var cachedPodCidr string - getRealPodCIDR := func() (string, error) { - if cachedPodCidr == "" { + var cachedPodCidrs []string + var cachedIpv4Cidr, cachedIpv6Cidr string + getRealPodCIDRs := func() (string, string, error) { + if len(cachedPodCidrs) == 0 { var err error - cachedPodCidr, err = getPodCidr(client, conf, epIDs.Node) + var emptyResult string + cachedPodCidrs, err = getPodCidrs(client, conf, epIDs.Node) if err != nil { - return "", err + return emptyResult, emptyResult, err + } + cachedIpv4Cidr, cachedIpv6Cidr, err = getIPsByFamily(cachedPodCidrs) + if err != nil { + return emptyResult, emptyResult, err } } - return cachedPodCidr, nil + return cachedIpv4Cidr, cachedIpv6Cidr, nil } - err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDR) + err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDRs) if err != nil { return nil, err } @@ -891,7 +897,9 @@ func getK8sPodInfo(client *kubernetes.Clientset, podName, podNamespace string) ( return labels, pod.Annotations, ports, profiles, generateName, serviceAccount, nil } -func getPodCidr(client *kubernetes.Clientset, conf types.NetConf, nodename string) (string, error) { +// getPodCidrs returns the podCidrs included in the node manifest +func getPodCidrs(client *kubernetes.Clientset, conf types.NetConf, nodename string) ([]string, error) { + var emptyString []string // Pull the node name out of the config if it's set. Defaults to nodename if conf.Kubernetes.NodeName != "" { nodename = conf.Kubernetes.NodeName @@ -899,11 +907,34 @@ func getPodCidr(client *kubernetes.Clientset, conf types.NetConf, nodename strin node, err := client.CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{}) if err != nil { - return "", err + return emptyString, err + } + if len(node.Spec.PodCIDRs) == 0 { + return emptyString, fmt.Errorf("no podCidr for node %s", nodename) + } + return node.Spec.PodCIDRs, nil +} + +// getIPsByFamily returns the IPv4 and IPv6 CIDRs +func getIPsByFamily(cidrs []string) (string, string, error) { + var ipv4Cidr, ipv6Cidr string + for _, cidr := range cidrs { + _, ipNet, err := cnet.ParseCIDR(cidr) + if err != nil { + return "", "", err + } + if ipNet.Version() == 4 { + ipv4Cidr = cidr + } + + if ipNet.Version() == 6 { + ipv6Cidr = cidr + } } - if node.Spec.PodCIDR == "" { - return "", fmt.Errorf("no podCidr for node %s", nodename) + if (len(cidrs) > 1) && (ipv4Cidr == "" || ipv6Cidr == "") { + return "", "", errors.New("ClusterCIDR contains two ranges of the same type") } - return node.Spec.PodCIDR, nil + + return ipv4Cidr, ipv6Cidr, nil } diff --git a/cni-plugin/tests/calico_cni_k8s_test.go b/cni-plugin/tests/calico_cni_k8s_test.go index d433a3bf22e..5b903e33edf 100644 --- a/cni-plugin/tests/calico_cni_k8s_test.go +++ b/cni-plugin/tests/calico_cni_k8s_test.go @@ -708,6 +708,52 @@ var _ = Describe("Kubernetes CNI tests", func() { numIPv4IPs: 1, numIPv6IPs: 1, }, + { + // This scenario tests IPv4+IPv6 without specifying any routes. + description: "new-style with IPv4 and IPv6 both using usePodCidr, no routes", + cniVersion: "0.3.0", + config: ` + { + "cniVersion": "%s", + "name": "net6", + "nodename_file_optional": true, + "type": "calico", + "etcd_endpoints": "http://%s:2379", + "datastore_type": "%s", + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "usePodCidr" + } + ], + [ + { + "subnet": "usePodCidrIPv6" + } + ] + ] + }, + "kubernetes": { + "kubeconfig": "/home/user/certs/kubeconfig" + }, + "policy": {"type": "k8s"}, + "log_level":"info" + }`, + expectedV4Routes: []string{ + regexp.QuoteMeta("default via 169.254.1.1 dev eth0"), + regexp.QuoteMeta("169.254.1.1 dev eth0 scope link"), + }, + expectedV6Routes: []string{ + "dead:beef::[0-9a-f]* dev eth0 proto kernel metric 256 pref medium", + "fe80::/64 dev eth0 proto kernel metric 256 pref medium", + "default via fe80::ecee:eeff:feee:eeee dev eth0 metric 1024", + }, + unexpectedRoute: regexp.QuoteMeta("10."), + numIPv4IPs: 1, + numIPv6IPs: 1, + }, { // In this scenario, we use a lot more of the host-local IPAM plugin. Namely: // - we use multiple ranges, one of which is IPv6, the other uses the podCIDR @@ -831,8 +877,13 @@ var _ = Describe("Kubernetes CNI tests", func() { }, } + // Run tests with PodCIDR for _, c := range hostLocalIPAMConfigs { c := c // Make sure we get a fresh variable on each loop. + // The dual-stack requires PodCIDRs + if strings.Contains(c.config, "usePodCidrIPv6") { + continue + } Context("Using host-local IPAM ("+c.description+"): request an IP then release it, and then request it again", func() { It("should successfully assign IP both times and successfully release it in the middle", func() { netconfHostLocalIPAM := fmt.Sprintf(c.config, c.cniVersion, os.Getenv("ETCD_IP"), os.Getenv("DATASTORE_TYPE")) @@ -942,6 +993,117 @@ var _ = Describe("Kubernetes CNI tests", func() { }) } + // Run tests with PodCIDRs defining a dual-stack deployment + for _, c := range hostLocalIPAMConfigs { + c := c // Make sure we get a fresh variable on each loop. + Context("Using host-local IPAM ("+c.description+"): request an IP then release it, and then request it again", func() { + It("should successfully assign IP both times and successfully release it in the middle", func() { + netconfHostLocalIPAM := fmt.Sprintf(c.config, c.cniVersion, os.Getenv("ETCD_IP"), os.Getenv("DATASTORE_TYPE")) + + clientset := getKubernetesClient() + + ensureNamespace(clientset, testutils.K8S_TEST_NS) + + ensureNodeDeleted(clientset, hostname) + + // Create a K8s Node object with PodCIDR and name equal to hostname. + _, err = clientset.CoreV1().Nodes().Create(context.Background(), &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: hostname}, + Spec: v1.NodeSpec{ + PodCIDRs: []string{"10.10.0.0/24", "dead:beef::/96"}, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer ensureNodeDeleted(clientset, hostname) + + By("Creating a pod with a specific IP address") + name := fmt.Sprintf("run%d", rand.Uint32()) + ensurePodCreated(clientset, testutils.K8S_TEST_NS, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: name, + Image: "ignore", + }}, + NodeName: hostname, + }, + }) + defer ensurePodDeleted(clientset, testutils.K8S_TEST_NS, name) + + requestedIP := "10.10.0.42" + expectedIP := net.IPv4(10, 10, 0, 42).To4() + + _, _, _, contAddresses, _, contNs, err := testutils.CreateContainer(netconfHostLocalIPAM, name, testutils.K8S_TEST_NS, requestedIP) + Expect(err).NotTo(HaveOccurred()) + + podIP := contAddresses[0].IP + Expect(podIP).Should(Equal(expectedIP)) + + By("Deleting the pod we created earlier") + _, err = testutils.DeleteContainer(netconfHostLocalIPAM, contNs.Path(), name, testutils.K8S_TEST_NS) + Expect(err).ShouldNot(HaveOccurred()) + + By("Creating a second pod with the same IP address as the first pod") + name2 := fmt.Sprintf("run2%d", rand.Uint32()) + ensurePodCreated(clientset, testutils.K8S_TEST_NS, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name2}, + Spec: v1.PodSpec{ + Containers: []v1.Container{{ + Name: fmt.Sprintf("container-%s", name2), + Image: "ignore", + }}, + NodeName: hostname, + }, + }) + defer ensurePodDeleted(clientset, testutils.K8S_TEST_NS, name2) + + _, _, _, contAddresses, _, contNs, err = testutils.CreateContainer(netconfHostLocalIPAM, name2, testutils.K8S_TEST_NS, requestedIP) + Expect(err).NotTo(HaveOccurred()) + + pod2IP := contAddresses[0].IP + Expect(pod2IP).Should(Equal(expectedIP)) + + err = contNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + out, err := exec.Command("ip", "route", "show").Output() + Expect(err).NotTo(HaveOccurred()) + for _, r := range c.expectedV4Routes { + Expect(string(out)).To(MatchRegexp(r)) + } + + if c.unexpectedRoute != "" { + Expect(string(out)).NotTo(ContainSubstring(c.unexpectedRoute)) + } + + out, err = exec.Command("ip", "-6", "route", "show").Output() + Expect(err).NotTo(HaveOccurred()) + for _, r := range c.expectedV6Routes { + Expect(string(out)).To(MatchRegexp(r)) + } + + if c.numIPv6IPs > 0 { + err := testutils.CheckSysctlValue("/proc/sys/net/ipv6/conf/eth0/accept_dad", "0") + Expect(err).NotTo(HaveOccurred()) + } + + out, err = exec.Command("ip", "addr", "show").Output() + Expect(err).NotTo(HaveOccurred()) + inet := regexp.MustCompile(` {4}inet .*scope global`) + Expect(inet.FindAll(out, -1)).To(HaveLen(c.numIPv4IPs)) + inetv6 := regexp.MustCompile(` {4}inet6 .*scope global`) + Expect(inetv6.FindAll(out, -1)).To(HaveLen(c.numIPv6IPs)) + Expect(out).NotTo(ContainSubstring("scope global tentative"), + "Some IPv6 addresses marked as tentative; disabling DAD must have failed.") + + return nil + }) + Expect(err).ShouldNot(HaveOccurred()) + + _, err = testutils.DeleteContainer(netconfHostLocalIPAM, contNs.Path(), name2, testutils.K8S_TEST_NS) + Expect(err).ShouldNot(HaveOccurred()) + }) + }) + } }) Context("using calico-ipam with a Namespace annotation only", func() {