diff --git a/cni-plugin/internal/pkg/utils/utils.go b/cni-plugin/internal/pkg/utils/utils.go index 8e7dc785e53..c0108668366 100644 --- a/cni-plugin/internal/pkg/utils/utils.go +++ b/cni-plugin/internal/pkg/utils/utils.go @@ -202,16 +202,18 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er // We need to replace "usePodCidr" with a valid, but dummy podCidr string with "host-local" IPAM. // host-local IPAM releases the IP by ContainerID, so podCidr isn't really used to release the IP. // It just needs a valid CIDR, but it doesn't have to be the CIDR associated with the host. - const dummyPodCidr = "0.0.0.0/0" + dummyPodCidrv4 := "0.0.0.0/0" + dummyPodCidrv6 := "::/0" var stdinData map[string]interface{} err := json.Unmarshal(args.StdinData, &stdinData) if err != nil { return err } - logger.WithField("podCidr", dummyPodCidr).Info("Using a dummy podCidr to release the IP") - getDummyPodCIDR := func() (string, error) { - return dummyPodCidr, nil + logger.WithFields(logrus.Fields{"podCidrv4": dummyPodCidrv4, + "podCidrv6": dummyPodCidrv6}).Info("Using dummy podCidrs to release the IPs") + getDummyPodCIDR := func() (string, string, error) { + return dummyPodCidrv4, dummyPodCidrv6, nil } err = ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getDummyPodCIDR) if err != nil { @@ -286,13 +288,13 @@ func DeleteIPAM(conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) er // } // ... // } -func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDR func() (string, error)) error { +func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]interface{}, getPodCIDRs func() (string, string, error)) error { ipamData, ok := stdinData["ipam"].(map[string]interface{}) if !ok { return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", stdinData["ipam"]) } // Older versions of host-local IPAM store a single subnet in the top-level IPAM dict. - err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDR) + err := replaceHostLocalIPAMPodCIDR(logger, ipamData, getPodCIDRs) if err != nil { return err } @@ -310,7 +312,7 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int return fmt.Errorf("failed to parse host-local IPAM range set; was expecting a list, not: %v", rs) } for _, r := range rs { - err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDR) + err := replaceHostLocalIPAMPodCIDR(logger, r, getPodCIDRs) if err != nil { return err } @@ -320,29 +322,41 @@ func ReplaceHostLocalIPAMPodCIDRs(logger *logrus.Entry, stdinData map[string]int return nil } -func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidr func() (string, error)) error { +func replaceHostLocalIPAMPodCIDR(logger *logrus.Entry, rawIpamData interface{}, getPodCidrs func() (string, string, error)) error { logrus.WithField("ipamData", rawIpamData).Debug("Examining IPAM data for usePodCidr") ipamData, ok := rawIpamData.(map[string]interface{}) if !ok { return fmt.Errorf("failed to parse host-local IPAM data; was expecting a dict, not: %v", rawIpamData) } subnet, _ := ipamData["subnet"].(string) + ipv4Cidr, ipv6Cidr, err := getPodCidrs() + if err != nil { + logger.Errorf("Failed to getPodCidrs") + return err + } + if strings.EqualFold(subnet, "usePodCidr") { - logger.Info("Calico CNI fetching podCidr from Kubernetes") - podCidr, err := getPodCidr() + if ipv4Cidr == "" { + return errors.New("usePodCidr found but there is no IPv4 CIDR configured") + } + ipamData["subnet"] = ipv4Cidr + subnet = ipv4Cidr + logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", ipv4Cidr) + + // updateHostLocalIPAMDataForOS is only required for Windows and only ipv4 is supported + err = updateHostLocalIPAMDataForOS(subnet, ipamData) if err != nil { - logger.Info("Failed to getPodCidr") return err } - logger.WithField("podCidr", podCidr).Info("Fetched podCidr") - ipamData["subnet"] = podCidr - subnet = podCidr - logger.Infof("Calico CNI passing podCidr to host-local IPAM: %s", podCidr) } - err := updateHostLocalIPAMDataForOS(subnet, ipamData) - if err != nil { - return err + if strings.EqualFold(subnet, "usePodCidrIPv6") { + if ipv6Cidr == "" { + return errors.New("usePodCidrIPv6 found but there is no IPv6 CIDR configured") + } + ipamData["subnet"] = ipv6Cidr + subnet = ipv6Cidr + logger.Infof("Calico CNI passing podCidrv6 to host-local IPAM: %s", ipv6Cidr) } return nil @@ -360,6 +374,7 @@ func UpdateHostLocalIPAMDataForWindows(subnet string, ipamData map[string]interf return err } //process only if we have ipv4 subnet + //VXLAN networks on Windows do not support dual-stack https://kubernetes.io/docs/setup/production-environment/windows/intro-windows-in-kubernetes/#ipv6-networking if ip.To4() != nil { //get Expected start and end range for given CIDR expStartRange, expEndRange := getIPRanges(ip, ipnet) diff --git a/cni-plugin/pkg/k8s/k8s.go b/cni-plugin/pkg/k8s/k8s.go index 866e7f6b7c4..05431977f05 100644 --- a/cni-plugin/pkg/k8s/k8s.go +++ b/cni-plugin/pkg/k8s/k8s.go @@ -105,18 +105,24 @@ func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, epID } // Defer to ReplaceHostLocalIPAMPodCIDRs to swap the "usePodCidr" value out. - var cachedPodCidr string - getRealPodCIDR := func() (string, error) { - if cachedPodCidr == "" { + var cachedPodCidrs []string + var cachedIpv4Cidr, cachedIpv6Cidr string + getRealPodCIDRs := func() (string, string, error) { + if len(cachedPodCidrs) == 0 { var err error - cachedPodCidr, err = getPodCidr(client, conf, epIDs.Node) + var emptyResult string + cachedPodCidrs, err = getPodCidrs(client, conf, epIDs.Node) if err != nil { - return "", err + return emptyResult, emptyResult, err + } + cachedIpv4Cidr, cachedIpv6Cidr, err = getIPsByFamily(cachedPodCidrs) + if err != nil { + return emptyResult, emptyResult, err } } - return cachedPodCidr, nil + return cachedIpv4Cidr, cachedIpv6Cidr, nil } - err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDR) + err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDRs) if err != nil { return nil, err } @@ -891,7 +897,9 @@ func getK8sPodInfo(client *kubernetes.Clientset, podName, podNamespace string) ( return labels, pod.Annotations, ports, profiles, generateName, serviceAccount, nil } -func getPodCidr(client *kubernetes.Clientset, conf types.NetConf, nodename string) (string, error) { +// getPodCidrs returns the podCidrs included in the node manifest +func getPodCidrs(client *kubernetes.Clientset, conf types.NetConf, nodename string) ([]string, error) { + var emptyString []string // Pull the node name out of the config if it's set. Defaults to nodename if conf.Kubernetes.NodeName != "" { nodename = conf.Kubernetes.NodeName @@ -899,11 +907,34 @@ func getPodCidr(client *kubernetes.Clientset, conf types.NetConf, nodename strin node, err := client.CoreV1().Nodes().Get(context.Background(), nodename, metav1.GetOptions{}) if err != nil { - return "", err + return emptyString, err + } + if len(node.Spec.PodCIDRs) == 0 { + return emptyString, fmt.Errorf("no podCidr for node %s", nodename) + } + return node.Spec.PodCIDRs, nil +} + +// getIPsByFamily returns the IPv4 and IPv6 CIDRs +func getIPsByFamily(cidrs []string) (string, string, error) { + var ipv4Cidr, ipv6Cidr string + for _, cidr := range cidrs { + _, ipNet, err := cnet.ParseCIDR(cidr) + if err != nil { + return "", "", err + } + if ipNet.Version() == 4 { + ipv4Cidr = cidr + } + + if ipNet.Version() == 6 { + ipv6Cidr = cidr + } } - if node.Spec.PodCIDR == "" { - return "", fmt.Errorf("no podCidr for node %s", nodename) + if (len(cidrs) > 1) && (ipv4Cidr == "" || ipv6Cidr == "") { + return "", "", errors.New("ClusterCIDR contains two ranges of the same type") } - return node.Spec.PodCIDR, nil + + return ipv4Cidr, ipv6Cidr, nil } diff --git a/cni-plugin/tests/calico_cni_k8s_test.go b/cni-plugin/tests/calico_cni_k8s_test.go index c7817cc1345..42b972a56fb 100644 --- a/cni-plugin/tests/calico_cni_k8s_test.go +++ b/cni-plugin/tests/calico_cni_k8s_test.go @@ -721,6 +721,52 @@ var _ = Describe("Kubernetes CNI tests", func() { numIPv4IPs: 1, numIPv6IPs: 1, }, + { + // This scenario tests IPv4+IPv6 without specifying any routes. + description: "new-style with IPv4 and IPv6 both using usePodCidr, no routes", + cniVersion: "0.3.0", + config: ` + { + "cniVersion": "%s", + "name": "net6", + "nodename_file_optional": true, + "type": "calico", + "etcd_endpoints": "http://%s:2379", + "datastore_type": "%s", + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "usePodCidr" + } + ], + [ + { + "subnet": "usePodCidrIPv6" + } + ] + ] + }, + "kubernetes": { + "kubeconfig": "/home/user/certs/kubeconfig" + }, + "policy": {"type": "k8s"}, + "log_level":"info" + }`, + expectedV4Routes: []string{ + regexp.QuoteMeta("default via 169.254.1.1 dev eth0"), + regexp.QuoteMeta("169.254.1.1 dev eth0 scope link"), + }, + expectedV6Routes: []string{ + "dead:beef::[0-9a-f]* dev eth0 proto kernel metric 256 pref medium", + "fe80::/64 dev eth0 proto kernel metric 256 pref medium", + "default via fe80::ecee:eeff:feee:eeee dev eth0 metric 1024", + }, + unexpectedRoute: regexp.QuoteMeta("10."), + numIPv4IPs: 1, + numIPv6IPs: 1, + }, { // In this scenario, we use a lot more of the host-local IPAM plugin. Namely: // - we use multiple ranges, one of which is IPv6, the other uses the podCIDR