From f25569c1b75f35aeb8588139478e25c5a53b7780 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Sun, 22 Aug 2021 16:10:33 -0700 Subject: [PATCH 1/7] IPv6 Support --- Makefile | 3 +- cmd/egress-v4-cni-plugin/cni.go | 433 +++++++++ cmd/egress-v4-cni-plugin/snat/snat.go | 127 +++ cmd/routed-eni-cni-plugin/cni.go | 39 +- cmd/routed-eni-cni-plugin/cni_test.go | 5 +- cmd/routed-eni-cni-plugin/driver/driver.go | 130 ++- .../driver/driver_test.go | 32 +- .../driver/mocks/driver_mocks.go | 14 +- misc/10-aws.conflist | 15 + pkg/awsutils/awsutils.go | 170 +++- pkg/awsutils/awsutils_test.go | 53 +- pkg/awsutils/imds.go | 24 +- pkg/awsutils/imds_test.go | 24 +- pkg/awsutils/mocks/awsutils_mocks.go | 64 +- pkg/ec2wrapper/client.go | 2 + pkg/ec2wrapper/mocks/ec2wrapper_mocks.go | 42 +- pkg/ipamd/datastore/data_store.go | 322 +++++-- pkg/ipamd/datastore/data_store_test.go | 10 +- pkg/ipamd/ipamd.go | 357 ++++++-- pkg/ipamd/ipamd_test.go | 364 ++++++-- pkg/ipamd/rpc_handler.go | 75 +- pkg/ipamd/rpc_handler_test.go | 109 ++- pkg/networkutils/mocks/network_mocks.go | 8 +- pkg/networkutils/network.go | 106 ++- pkg/networkutils/network_test.go | 79 +- rpc/rpc.pb.go | 829 ++++++++++-------- rpc/rpc.proto | 17 +- rpc/rpc_grpc.pb.go | 136 +++ scripts/dockerfiles/Dockerfile.release | 2 + scripts/dockerfiles/Dockerfile.test | 1 - scripts/entrypoint.sh | 24 +- 31 files changed, 2768 insertions(+), 848 deletions(-) create mode 100644 cmd/egress-v4-cni-plugin/cni.go create mode 100644 cmd/egress-v4-cni-plugin/snat/snat.go create mode 100644 rpc/rpc_grpc.pb.go diff --git a/Makefile b/Makefile index 57226f4ca3..a6d22b9615 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper # Plugin binaries # Not copied: bridge dhcp firewall flannel host-device host-local ipvlan macvlan ptp sbr static tuning vlan # For gnu tar, the full path in the tar file is required -PLUGIN_BINS = ./loopback ./portmap ./bandwidth +PLUGIN_BINS = ./loopback ./portmap ./bandwidth ./host-local # DOCKER_ARGS is extra arguments passed during container image build. DOCKER_ARGS = @@ -120,6 +120,7 @@ build-linux: ## Build the VPC CNI plugin agent using the host's Go toolchain. go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o aws-k8s-agent ./cmd/aws-k8s-agent go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o aws-cni ./cmd/routed-eni-cni-plugin go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o grpc-health-probe ./cmd/grpc-health-probe + go build $(VENDOR_OVERRIDE_FLAG) $(BUILD_FLAGS) -o egress-v4-cni ./cmd/egress-v4-cni-plugin # Build VPC CNI plugin & agent container image. docker: setup-ec2-sdk-override ## Build VPC CNI plugin & agent container image. diff --git a/cmd/egress-v4-cni-plugin/cni.go b/cmd/egress-v4-cni-plugin/cni.go new file mode 100644 index 0000000000..b8c31d4bbb --- /dev/null +++ b/cmd/egress-v4-cni-plugin/cni.go @@ -0,0 +1,433 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger" + "net" + "os" + "runtime" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + cniversion "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ip" + "github.com/containernetworking/plugins/pkg/ipam" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/utils" + "github.com/vishvananda/netlink" + "github.com/containernetworking/plugins/pkg/utils/sysctl" + + "github.com/aws/amazon-vpc-cni-k8s/cmd/egress-v4-cni-plugin/snat" +) + +var version string + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +// NetConf is our CNI config structure +type NetConf struct { + types.NetConf + + // Interface inside container to create + IfName string `json:"ifName"` + + //MTU for Egress v4 interface + MTU int `json:"mtu"` + + Enabled string `json:"enabled"` + + // IP to use as SNAT target + NodeIP net.IP `json:"nodeIP"` + + PluginLogFile string `json:"pluginLogFile"` + PluginLogLevel string `json:"pluginLogLevel"` +} + +func loadConf(bytes []byte) (*NetConf, logger.Logger, error) { + conf := &NetConf{IfName: "v4if0"} + + if err := json.Unmarshal(bytes, conf); err != nil { + return nil, nil, err + } + + if conf.RawPrevResult != nil { + if err := cniversion.ParsePrevResult(&conf.NetConf); err != nil { + return nil, nil, fmt.Errorf("could not parse prevResult: %v", err) + } + } + + logConfig := logger.Configuration{ + LogLevel: conf.PluginLogLevel, + LogLocation: conf.PluginLogFile, + } + log := logger.New(&logConfig) + return conf, log, nil +} + +// The bulk of this file is mostly based on standard ptp CNI plugin. +// +// Note: There are other options, for example we could add a new +// address onto an existing container/host interface. +// +// Unfortunately kubelet's dockershim (at least) ignores the CNI +// result structure, and directly queries the addresses on the +// container's IfName - and then prefers any global v4 address found. +// We do _not_ want our v4 NAT address to become "the" pod IP! +// +// Also, standard `loopback` CNI plugin checks and aborts if it finds +// any global-scope addresses on `lo`, so we can't just do that +// either. +// +// So we have to create a new interface (not args.IfName) to hide our +// NAT address from all this logic (or patch dockershim, or (better) +// just stop using dockerd...). Hence ptp. +// + +func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Result) (*current.Interface, *current.Interface, error) { + // The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1. + // What we want is really a point-to-point link but veth does not support IFF_POINTTOPOINT. + // Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and + // add a route like "192.168.3.0/24 via 192.168.3.1 dev $ifName". + // Unfortunately that won't work as the GW will be outside the interface's subnet. + + // Our solution is to configure the interface with 192.168.3.5/24, then delete the + // "192.168.3.0/24 dev $ifName" route that was automatically added. Then we add + // "192.168.3.1/32 dev $ifName" and "192.168.3.0/24 via 192.168.3.1 dev $ifName". + // In other words we force all traffic to ARP via the gateway except for GW itself. + + hostInterface := ¤t.Interface{} + containerInterface := ¤t.Interface{} + + err := netns.Do(func(hostNS ns.NetNS) error { + hostVeth, contVeth0, err := ip.SetupVeth(ifName, mtu, hostNS) + if err != nil { + return err + } + hostInterface.Name = hostVeth.Name + hostInterface.Mac = hostVeth.HardwareAddr.String() + containerInterface.Name = contVeth0.Name + containerInterface.Mac = contVeth0.HardwareAddr.String() + containerInterface.Sandbox = netns.Path() + + for _, ipc := range pr.IPs { + // All addresses apply to the container veth interface + ipc.Interface = current.Int(1) + } + + pr.Interfaces = []*current.Interface{hostInterface, containerInterface} + + if err = ipam.ConfigureIface(ifName, pr); err != nil { + return err + } + + contVeth, err := net.InterfaceByName(ifName) + if err != nil { + return fmt.Errorf("failed to look up %q: %v", ifName, err) + } + + for _, ipc := range pr.IPs { + // Delete the route that was automatically added + route := netlink.Route{ + LinkIndex: contVeth.Index, + Dst: &net.IPNet{ + IP: ipc.Address.IP.Mask(ipc.Address.Mask), + Mask: ipc.Address.Mask, + }, + Scope: netlink.SCOPE_NOWHERE, + } + + if err := netlink.RouteDel(&route); err != nil { + return fmt.Errorf("failed to delete route %v: %v", route, err) + } + + addrBits := 32 + if ipc.Version == "6" { + addrBits = 128 + } + + for _, r := range []netlink.Route{ + { + LinkIndex: contVeth.Index, //TODO - Should be default route + Dst: &net.IPNet{ + IP: ipc.Gateway, + Mask: net.CIDRMask(addrBits, addrBits), + }, + Scope: netlink.SCOPE_LINK, + Src: ipc.Address.IP, + }, + { + LinkIndex: contVeth.Index, + Dst: &net.IPNet{ + IP: ipc.Address.IP.Mask(ipc.Address.Mask), + Mask: ipc.Address.Mask, + }, + Scope: netlink.SCOPE_UNIVERSE, + Gw: ipc.Gateway, + Src: ipc.Address.IP, + }, + } { + if err := netlink.RouteAdd(&r); err != nil { + return fmt.Errorf("failed to add route %v: %v", r, err) + } + } + } + + //Disable IPv6 on this interface + _, err = sysctl.Sysctl("net/ipv6/conf/"+ifName+"/disable_ipv6", "1") + if err != nil { + return fmt.Errorf("failed to disable IPv6 for interface: %v", err) + } + //Block traffic directed to 169.254.172.0/22 from the Pod + err = snat.SetupRuleToBlockNodeLocalV4Access() + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, nil, err + } + return hostInterface, containerInterface, nil +} + +func setupHostVeth(vethName string, result *current.Result) error { + // hostVeth moved namespaces and may have a new ifindex + veth, err := netlink.LinkByName(vethName) + if err != nil { + return fmt.Errorf("failed to lookup %q: %v", vethName, err) + } + + for _, ipc := range result.IPs { + maskLen := 128 + if ipc.Address.IP.To4() != nil { + maskLen = 32 + } + + // NB: this is modified from standard ptp plugin. + + ipn := &net.IPNet{ + IP: ipc.Gateway, + Mask: net.CIDRMask(maskLen, maskLen), + } + addr := &netlink.Addr{ + IPNet: ipn, + Scope: int(netlink.SCOPE_LINK), // <- ptp uses SCOPE_UNIVERSE here + } + if err = netlink.AddrAdd(veth, addr); err != nil { + return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err) + } + + ipn = &net.IPNet{ + IP: ipc.Address.IP, + Mask: net.CIDRMask(maskLen, maskLen), + } + err := netlink.RouteAdd(&netlink.Route{ + LinkIndex: veth.Attrs().Index, + Scope: netlink.SCOPE_LINK, // <- ptp uses SCOPE_HOST here + Dst: ipn, + }) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to add route on host: %v", err) + } + } + + return nil +} + +func main() { + skel.PluginMain(cmdAdd, nil, cmdDel, cniversion.All, fmt.Sprintf("egress-v4 CNI plugin %s", version)) +} + +func cmdAdd(args *skel.CmdArgs) error { + netConf, log, err := loadConf(args.StdinData) + if err != nil { + log.Debugf("Received Add request: Failed to parse config") + return fmt.Errorf("failed to parse config: %v", err) + } + + if netConf.PrevResult == nil { + return fmt.Errorf("must be called as a chained plugin") + } + + result, err := current.GetResult(netConf.PrevResult) + if err != nil { + return err + } + + log.Debugf("Received an ADD request for: conf=%v; Plugin enabled=%s", netConf, netConf.Enabled) + //We will not be vending out this as a separate plugin by itself and it is only intended to be used as a + //chained plugin to VPC CNI in IPv6 mode. We only need this plugin to kick in if v6 is enabled in VPC CNI. So, the + //value of an env variable in VPC CNI determines whether this plugin should be enabled and this is an attempt to + //pass through the variable configured in VPC CNI. + if netConf.Enabled == "false" { + return types.PrintResult(result, netConf.CNIVersion) + } + + chain := utils.MustFormatChainNameWithPrefix(netConf.Name, args.ContainerID, "E4-") + comment := utils.FormatComment(netConf.Name, args.ContainerID) + + ipamResultI, err := ipam.ExecAdd(netConf.IPAM.Type, args.StdinData) + if err != nil { + return fmt.Errorf("running IPAM plugin failed: %v", err) + } + + // Invoke ipam del if err to avoid ip leak + defer func() { + if err != nil { + ipam.ExecDel(netConf.IPAM.Type, args.StdinData) + } + }() + + tmpResult, err := current.NewResultFromResult(ipamResultI) + if err != nil { + return err + } + + if len(tmpResult.IPs) == 0 { + return fmt.Errorf("IPAM plugin returned zero IPs") + } + + if err := ip.EnableForward(tmpResult.IPs); err != nil { + return fmt.Errorf("could not enable IP forwarding: %v", err) + } + + netns, err := ns.GetNS(args.Netns) + if err != nil { + return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) + } + defer netns.Close() + + // NB: This uses netConf.IfName NOT args.IfName. + hostInterface, _, err := setupContainerVeth(netns, netConf.IfName, netConf.MTU, tmpResult) + if err != nil { + log.Debugf("failed to setup container Veth: %v", err) + return err + } + + if err = setupHostVeth(hostInterface.Name, tmpResult); err != nil { + return err + } + + log.Debugf("Node IP: %s", netConf.NodeIP) + if netConf.NodeIP != nil { + for _, ipc := range tmpResult.IPs { + if ipc.Version == "4" { + //log.Printf("Configuring SNAT %s -> %s", ipc.Address.IP, netConf.SnatIP) + if err := snat.Snat4(netConf.NodeIP, ipc.Address.IP, chain, comment); err != nil { + return err + } + } + } + } + + //Copy interfaces over to result, but not IPs. + result.Interfaces = append(result.Interfaces, tmpResult.Interfaces...) + //Note: Useful for debug, will do away with the below log prior to release + for _,v := range result.IPs { + log.Debugf("Interface Name: %v; IP: %s", v.Interface, v.Address) + } + + // Pass through the previous result + return types.PrintResult(result, netConf.CNIVersion) +} + +func cmdDel(args *skel.CmdArgs) error { + netConf, log, err := loadConf(args.StdinData) + if err != nil { + return fmt.Errorf("failed to parse config: %v", err) + } + + //We only need this plugin to kick in if v6 is enabled + if netConf.Enabled == "false" { + return nil + } + + log.Debugf("Received Del Request: conf=%v", netConf) + if err := ipam.ExecDel(netConf.IPAM.Type, args.StdinData); err != nil { + log.Debugf("running IPAM plugin failed: %v", err) + return fmt.Errorf("running IPAM plugin failed: %v", err) + } + + ipnets := []*net.IPNet{} + if args.Netns != "" { + err := ns.WithNetNSPath(args.Netns, func(hostNS ns.NetNS) error { + var err error + + // DelLinkByNameAddr function deletes an interface and returns IPs assigned to it but it + // excludes IPs that are not global unicast addresses (or) private IPs. Will not work for + // our scenario as we use 169.254.0.0/16 range for v4 IPs. + + //Get the interface we want to delete + iface, err := netlink.LinkByName(netConf.IfName) + + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + return nil + } + return nil + } + + //Retrieve IP addresses assigned to the interface + addrs, err := netlink.AddrList(iface, netlink.FAMILY_ALL) + if err != nil { + return fmt.Errorf("failed to get IP addresses for %q: %v", netConf.IfName, err) + } + + //Delete the interface/link. + if err = netlink.LinkDel(iface); err != nil { + return fmt.Errorf("failed to delete %q: %v", netConf.IfName, err) + } + + for _, addr := range addrs { + ipnets = append(ipnets, addr.IPNet) + } + + if err != nil && err == ip.ErrLinkNotFound { + log.Debugf("DEL: Link Not Found, returning", err) + return nil + } + return err + }) + + //DEL should be best effort. We should clean up as much as we can and avoid returning error + //CNI Spec: TODO + if err != nil { + log.Debugf("DEL: Executing in container ns errored out, returning", err) + return err + } + } + + chain := utils.MustFormatChainNameWithPrefix(netConf.Name, args.ContainerID, "E4-") + comment := utils.FormatComment(netConf.Name, args.ContainerID) + + if netConf.NodeIP != nil { + log.Debugf("DEL: SNAT setup, let's clean them up. Size of ipnets: %d", len(ipnets)) + for _, ipn := range ipnets { + if err := snat.Snat4Del(ipn.IP, chain, comment); err != nil { + return err + } + } + } + + return nil +} diff --git a/cmd/egress-v4-cni-plugin/snat/snat.go b/cmd/egress-v4-cni-plugin/snat/snat.go new file mode 100644 index 0000000000..b6a148a360 --- /dev/null +++ b/cmd/egress-v4-cni-plugin/snat/snat.go @@ -0,0 +1,127 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package snat + +import ( + "fmt" + "net" + + "github.com/coreos/go-iptables/iptables" +) + +func iptRules4(target, src net.IP, chain, comment string, useRandomFully bool) [][]string { + var rules [][]string + + // Accept/ignore multicast (just because we can) + rules = append(rules, []string{chain, "-d", "224.0.0.0/4", "-j", "ACCEPT", "-m", "comment", "--comment", comment}) + + // SNAT + args := []string{ + chain, + "-j", "SNAT", + "--to-source", target.String(), + "-m", "comment", "--comment", comment, + } + if useRandomFully { + args = append(args, "--random-fully") + } + rules = append(rules, args) + rules = append(rules, []string{"POSTROUTING", "-s", src.String(), "-j", chain, "-m", "comment", "--comment", comment}) + + return rules +} + +//Setup a rule to block egress traffic directed to 169.254.172.0/22 from the Pod +func SetupRuleToBlockNodeLocalV4Access() error{ + ipt, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err := ipt.AppendUnique("filter", "OUTPUT", "-d", "169.254.172.0/22", "-m", "comment", + "--comment", "Block Node Local Pod access via IPv4", "-j", "DROP"); err != nil { + return fmt.Errorf("failed adding v4 drop route: %v", err) + } + + return nil +} + +// Snat4 SNATs IPv4 connections from `src` to `target` +func Snat4(target, src net.IP, chain, comment string) error { + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return fmt.Errorf("failed to locate iptables: %v", err) + } + + rules := iptRules4(target, src, chain, comment, ipt.HasRandomFully()) + + chains, err := ipt.ListChains("nat") + if err != nil { + return err + } + existingChains := make(map[string]bool, len(chains)) + for _, ch := range chains { + existingChains[ch] = true + } + + for _, rule := range rules { + chain := rule[0] + if !existingChains[chain] { + if err = ipt.NewChain("nat", chain); err != nil { + return err + } + existingChains[chain] = true + } + } + + for _, rule := range rules { + chain := rule[0] + if err := ipt.AppendUnique("nat", chain, rule[1:]...); err != nil { + return err + } + } + + return nil +} + +// Snat4Del removes rules added by snat4 +func Snat4Del(src net.IP, chain, comment string) error { + ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) + if err != nil { + return fmt.Errorf("failed to locate iptables: %v", err) + } + + err = ipt.Delete("nat", "POSTROUTING", "-s", src.String(), "-j", chain, "-m", "comment", "--comment", comment) + if err != nil && !isNotExist(err) { + return err + } + + err = ipt.ClearChain("nat", chain) + if err != nil && !isNotExist(err) { + return err + } + + err = ipt.DeleteChain("nat", chain) + if err != nil && !isNotExist(err) { + return err + } + + return nil +} + +// isNotExist returns true if the error is from iptables indicating +// that the target does not exist. +func isNotExist(err error) bool { + e, ok := err.(*iptables.Error) + if !ok { + return false + } + return e.IsNotExist() +} diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index d6c320282b..c563c4e832 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -19,11 +19,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "net" - "os" - "runtime" - "strings" - "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -31,6 +26,10 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" "google.golang.org/grpc" + "net" + "os" + "runtime" + "strings" "github.com/aws/amazon-vpc-cni-k8s/cmd/routed-eni-cni-plugin/driver" "github.com/aws/amazon-vpc-cni-k8s/pkg/grpcwrapper" @@ -170,23 +169,39 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received add network response for container %s interface %s: %+v", args.ContainerID, args.IfName, r) - addr := &net.IPNet{ - IP: net.ParseIP(r.IPv4Addr), - Mask: net.IPv4Mask(255, 255, 255, 255), + //We will let the values in result struct guide us in terms of IP Address Family configured. + var v4Addr, v6Addr, addr *net.IPNet + var addrFamily string + + //We don't support dual stack mode currently so it has to be either v4 or v6 mode. + if r.IPv4Addr != "" { + v4Addr = &net.IPNet{ + IP: net.ParseIP(r.IPv4Addr), + Mask: net.CIDRMask(32, 32), + } + addrFamily = "4" + addr = v4Addr + } else if r.IPv6Addr != "" { + v6Addr = &net.IPNet{ + IP: net.ParseIP(r.IPv6Addr), + Mask: net.CIDRMask(128, 128), + } + addrFamily = "6" + addr = v6Addr } var hostVethName string if r.PodVlanId != 0 { hostVethName = generateHostVethName("vlan", string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, addr, int(r.PodVlanId), r.PodENIMAC, + err = driverClient.SetupPodENINetwork(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.PodVlanId), r.PodENIMAC, r.PodENISubnetGW, int(r.ParentIfIndex), mtu, log) } else { // build hostVethName // Note: the maximum length for linux interface name is 15 hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, addr, int(r.DeviceNumber), r.VPCcidrs, r.UseExternalSNAT, mtu, log) + err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCV4Cidrs, r.UseExternalSNAT, mtu, log) } if err != nil { @@ -217,8 +232,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap containerInterfaceIndex := 1 ips := []*current.IPConfig{ { - Version: "4", - Address: *addr, + Version: addrFamily, + Address: *addr, Interface: &containerInterfaceIndex, }, } diff --git a/cmd/routed-eni-cni-plugin/cni_test.go b/cmd/routed-eni-cni-plugin/cni_test.go index 2cdce073df..bb6ea04828 100644 --- a/cmd/routed-eni-cni-plugin/cni_test.go +++ b/cmd/routed-eni-cni-plugin/cni_test.go @@ -91,13 +91,14 @@ func TestCmdAdd(t *testing.T) { addNetworkReply := &rpc.AddNetworkReply{Success: true, IPv4Addr: ipAddr, DeviceNumber: devNum} mockC.EXPECT().AddNetwork(gomock.Any(), gomock.Any()).Return(addNetworkReply, nil) - addr := &net.IPNet{ + v4Addr := &net.IPNet{ IP: net.ParseIP(addNetworkReply.IPv4Addr), Mask: net.IPv4Mask(255, 255, 255, 255), } + v6Addr := &net.IPNet{} mocksNetwork.EXPECT().SetupNS(gomock.Any(), cmdArgs.IfName, cmdArgs.Netns, - addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) + v4Addr, v6Addr, int(addNetworkReply.DeviceNumber), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil) mocksTypes.EXPECT().PrintResult(gomock.Any(), gomock.Any()).Return(nil) diff --git a/cmd/routed-eni-cni-plugin/driver/driver.go b/cmd/routed-eni-cni-plugin/driver/driver.go index 78440ea181..63d0fed93e 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver.go +++ b/cmd/routed-eni-cni-plugin/driver/driver.go @@ -47,9 +47,9 @@ const ( // NetworkAPIs defines network API calls type NetworkAPIs interface { - SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error + SetupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error TeardownNS(addr *net.IPNet, deviceNumber int, log logger.Logger) error - SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, vlanID int, eniMAC string, + SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, vlanID int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error TeardownPodENINetwork(vlanID int, log logger.Logger) error } @@ -74,20 +74,24 @@ func New() NetworkAPIs { type createVethPairContext struct { contVethName string hostVethName string - addr *net.IPNet + v4Addr *net.IPNet + v6Addr *net.IPNet netLink netlinkwrapper.NetLink ip ipwrapper.IP mtu int + procSys procsyswrapper.ProcSys } -func newCreateVethPairContext(contVethName string, hostVethName string, addr *net.IPNet, mtu int) *createVethPairContext { +func newCreateVethPairContext(contVethName string, hostVethName string, v4Addr *net.IPNet, v6Addr *net.IPNet, mtu int) *createVethPairContext { return &createVethPairContext{ contVethName: contVethName, hostVethName: hostVethName, - addr: addr, + v4Addr: v4Addr, + v6Addr: v6Addr, netLink: netlinkwrapper.NewNetLink(), ip: ipwrapper.NewIP(), mtu: mtu, + procSys: procsyswrapper.NewProcSys(), } } @@ -128,12 +132,53 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { return errors.Wrapf(err, "setup NS network: failed to set link %q up", createVethContext.contVethName) } - // Add a connected route to a dummy next hop (169.254.1.1) + if createVethContext.v6Addr != nil && createVethContext.v6Addr.IP.To16() != nil { + //Enable v6 support on Container's veth interface. + if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", createVethContext.contVethName), "0"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container veth interface") + } + } + + //Enable v6 support on Container's lo interface inside the Pod networking namespace. + if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/lo/disable_ipv6"), "0"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container's lo interface") + } + } + + //Enable v6 forwarding on Container's veth interface. We will set the same on Host side veth after we move it to host netns. + if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/forwarding", createVethContext.contVethName), "1"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 forwarding container veth interface") + } + } + } + + // Add a connected route to a dummy next hop (169.254.1.1 or fe80::1) // # ip route show // default via 169.254.1.1 dev eth0 // 169.254.1.1 dev eth0 - gw := net.IPv4(169, 254, 1, 1) - gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)} + + var gwIP string + var maskLen int + var addr *netlink.Addr + var defNet *net.IPNet + + if createVethContext.v4Addr != nil { + gwIP = "169.254.1.1" + maskLen = 32 + addr = &netlink.Addr{IPNet: createVethContext.v4Addr} + _, defNet, _ = net.ParseCIDR("0.0.0.0/0") + } else if createVethContext.v6Addr != nil { + gwIP = "fe80::1" + maskLen = 128 + addr = &netlink.Addr{IPNet: createVethContext.v6Addr} + _, defNet, _ = net.ParseCIDR("::/0") + } + + gw := net.ParseIP(gwIP) + gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(maskLen, maskLen)} if err = createVethContext.netLink.RouteReplace(&netlink.Route{ LinkIndex: contVeth.Attrs().Index, @@ -142,18 +187,24 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { return errors.Wrap(err, "setup NS network: failed to add default gateway") } - // Add a default route via dummy next hop(169.254.1.1). Then all outgoing traffic will be routed by this - // default route via dummy next hop (169.254.1.1). - if err = createVethContext.ip.AddDefaultRoute(gwNet.IP, contVeth); err != nil { + // Add a default route via dummy next hop(169.254.1.1 or fe80::1). Then all outgoing traffic will be routed by this + // default route via dummy next hop (169.254.1.1 or fe80::1) + if err = createVethContext.netLink.RouteAdd(&netlink.Route{ + LinkIndex: contVeth.Attrs().Index, + Scope: netlink.SCOPE_UNIVERSE, + Dst: defNet, + Gw: gw, + }); err != nil { return errors.Wrap(err, "setup NS network: failed to add default route") } - if err = createVethContext.netLink.AddrAdd(contVeth, &netlink.Addr{IPNet: createVethContext.addr}); err != nil { + if err = createVethContext.netLink.AddrAdd(contVeth, addr); err != nil { return errors.Wrapf(err, "setup NS network: failed to add IP addr to %q", createVethContext.contVethName) } // add static ARP entry for default gateway // we are using routed mode on the host and container need this static ARP entry to resolve its default gateway. + // IP address family is derived from the IP address passed to the function (v4 or v6) neigh := &netlink.Neigh{ LinkIndex: contVeth.Attrs().Index, State: netlink.NUD_PERMANENT, @@ -174,23 +225,33 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { } // SetupNS wires up linux networking for a pod's network -func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error { +func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, + deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error { log.Debugf("SetupNS: hostVethName=%s, contVethName=%s, netnsPath=%s, deviceNumber=%d, mtu=%d", hostVethName, contVethName, netnsPath, deviceNumber, mtu) - return setupNS(hostVethName, contVethName, netnsPath, addr, deviceNumber, vpcCIDRs, useExternalSNAT, os.netLink, os.ns, mtu, log, os.procSys) + return setupNS(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, deviceNumber, vpcCIDRs, useExternalSNAT, os.netLink, os.ns, mtu, log, os.procSys) } -func setupNS(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, +func setupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int, log logger.Logger, procSys procsyswrapper.ProcSys) error { - hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, addr, netLink, ns, mtu, procSys, log) + hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, netLink, ns, mtu, procSys, log) if err != nil { return errors.Wrapf(err, "setupNS network: failed to setup veth pair.") } log.Debugf("Setup host route outgoing hostVeth, LinkIndex %d", hostVeth.Attrs().Index) - addrHostAddr := &net.IPNet{ - IP: addr.IP, - Mask: net.CIDRMask(32, 32)} + + var addrHostAddr *net.IPNet + //We only support either v4 or v6 modes. + if v4Addr != nil && v4Addr.IP.To4() != nil { + addrHostAddr = &net.IPNet{ + IP: v4Addr.IP, + Mask: net.CIDRMask(32, 32)} + } else if v6Addr != nil && v6Addr.IP.To16() != nil { + addrHostAddr = &net.IPNet{ + IP: v6Addr.IP, + Mask: net.CIDRMask(128, 128)} + } // Add host route route := netlink.Route{ @@ -204,32 +265,32 @@ func setupNS(hostVethName string, contVethName string, netnsPath string, addr *n } log.Debugf("Successfully set host route to be %s/0", route.Dst.IP.String()) - err = addContainerRule(netLink, true, addr, mainRouteTable) + err = addContainerRule(netLink, true, addrHostAddr, mainRouteTable) if err != nil { - log.Errorf("Failed to add toContainer rule for %s err=%v, ", addr.String(), err) + log.Errorf("Failed to add toContainer rule for %s err=%v, ", addrHostAddr.String(), err) return errors.Wrap(err, "setupNS network: failed to add toContainer") } - log.Infof("Added toContainer rule for %s", addr.String()) + log.Infof("Added toContainer rule for %s", addrHostAddr.String()) // add from-pod rule, only need it when it is not primary ENI if deviceNumber > 0 { // To be backwards compatible, we will have to keep this off-by one setting tableNumber := deviceNumber + 1 // add rule: 1536: from use table - err = addContainerRule(netLink, false, addr, tableNumber) + err = addContainerRule(netLink, false, addrHostAddr, tableNumber) if err != nil { - log.Errorf("Failed to add fromContainer rule for %s err: %v", addr.String(), err) + log.Errorf("Failed to add fromContainer rule for %s err: %v", addrHostAddr.String(), err) return errors.Wrap(err, "add NS network: failed to add fromContainer rule") } - log.Infof("Added rule priority %d from %s table %d", fromContainerRulePriority, addr.String(), tableNumber) + log.Infof("Added rule priority %d from %s table %d", fromContainerRulePriority, addrHostAddr.String(), tableNumber) } return nil } // setupVeth sets up veth for the pod. -func setupVeth(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, netLink netlinkwrapper.NetLink, +func setupVeth(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int, procSys procsyswrapper.ProcSys, log logger.Logger) (netlink.Link, error) { // Clean up if hostVeth exists. if oldHostVeth, err := netLink.LinkByName(hostVethName); err == nil { @@ -239,7 +300,8 @@ func setupVeth(hostVethName string, contVethName string, netnsPath string, addr log.Debugf("Cleaned up old hostVeth: %v\n", hostVethName) } - createVethContext := newCreateVethPairContext(contVethName, hostVethName, addr, mtu) + log.Debugf("v4addr: %v; v6Addr: %v\n", v4Addr, v6Addr) + createVethContext := newCreateVethPairContext(contVethName, hostVethName, v4Addr, v6Addr, mtu) if err := ns.WithNetNSPath(netnsPath, createVethContext.run); err != nil { log.Errorf("Failed to setup veth network %v", err) return nil, errors.Wrap(err, "setupVeth network: failed to setup veth network") @@ -250,7 +312,6 @@ func setupVeth(hostVethName string, contVethName string, netnsPath string, addr return nil, errors.Wrapf(err, "setupVeth network: failed to find link %q", hostVethName) } - // NB: Must be set after move to host namespace, or kernel will reset to defaults. if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0"); err != nil { if !os.IsNotExist(err) { return nil, errors.Wrapf(err, "setupVeth network: failed to disable IPv6 router advertisements") @@ -275,10 +336,10 @@ func setupVeth(hostVethName string, contVethName string, netnsPath string, addr } // SetupPodENINetwork sets up the network ns for pods requesting its own security group -func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, addr *net.IPNet, - vlanID int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error { +func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, + v6Addr *net.IPNet, vlanID int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error { - hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, addr, os.netLink, os.ns, mtu, os.procSys, log) + hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, os.netLink, os.ns, mtu, os.procSys, log) if err != nil { return errors.Wrapf(err, "SetupPodENINetwork failed to setup veth pair.") } @@ -327,6 +388,13 @@ func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName str } } + var addr *net.IPNet + if v4Addr != nil { + addr = v4Addr + } else if v6Addr != nil { + addr = v6Addr + } + // 5. create route entry for hostveth. route := netlink.Route{ LinkIndex: hostVeth.Attrs().Index, diff --git a/cmd/routed-eni-cni-plugin/driver/driver_test.go b/cmd/routed-eni-cni-plugin/driver/driver_test.go index f365e2975e..839ed6e553 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver_test.go +++ b/cmd/routed-eni-cni-plugin/driver/driver_test.go @@ -39,6 +39,7 @@ import ( const ( testMAC = "01:23:45:67:89:ab" testIP = "10.0.10.10" + testV6IP = "2001:db8::1" testContVethName = "eth0" testHostVethName = "aws-eth0" testVlanName = "vlan.eth.1" @@ -82,7 +83,7 @@ func (m *testMocks) mockWithFailureAt(t *testing.T, failAt string) *createVethPa hostVethName: testHostVethName, netLink: m.netlink, ip: m.ip, - addr: &net.IPNet{ + v4Addr: &net.IPNet{ IP: net.ParseIP(testIP), Mask: net.IPv4Mask(255, 255, 255, 255), }, @@ -359,7 +360,22 @@ func TestSetupPodNetwork(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, true, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, true, m.netlink, m.ns, mtu, log, m.procsys) + assert.NoError(t, err) +} + +func TestSetupIPv6PodNetwork(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + m.mockSetupPodNetworkWithFailureAt(t, "") + v6Addr := &net.IPNet{ + IP: net.ParseIP(testV6IP), + Mask: net.CIDRMask(128, 128), + } + + var cidrs []string + err := setupNS(testHostVethName, testContVethName, testnetnsPath, &net.IPNet{}, v6Addr, testTable, cidrs, true, m.netlink, m.ns, mtu, log, m.procsys) assert.NoError(t, err) } @@ -374,7 +390,7 @@ func TestSetupPodNetworkErrNoIPv6(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, true, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, true, m.netlink, m.ns, mtu, log, m.procsys) assert.NoError(t, err) } @@ -389,7 +405,7 @@ func TestSetupPodNetworkErrLinkByName(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) assert.Error(t, err) } @@ -405,7 +421,7 @@ func TestSetupPodNetworkErrLinkSetup(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) assert.Error(t, err) } @@ -421,7 +437,7 @@ func TestSetupPodNetworkErrProcSys(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) assert.Error(t, err) } @@ -437,7 +453,7 @@ func TestSetupPodNetworkErrRouteReplace(t *testing.T) { Mask: net.IPv4Mask(255, 255, 255, 255), } var cidrs []string - err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) + err := setupNS(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, testTable, cidrs, false, m.netlink, m.ns, mtu, log, m.procsys) assert.Error(t, err) } @@ -587,7 +603,7 @@ func TestSetupPodENINetworkHappyCase(t *testing.T) { m.mockSetupPodENINetworkWithFailureAt(t, addr, "") - err := t1.SetupPodENINetwork(testHostVethName, testContVethName, testnetnsPath, addr, 1, "eniMacAddress", + err := t1.SetupPodENINetwork(testHostVethName, testContVethName, testnetnsPath, addr, &net.IPNet{}, 1, "eniMacAddress", "10.1.0.1", 2, mtu, log) assert.NoError(t, err) diff --git a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go index 0b8248c078..1702f3a812 100644 --- a/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go +++ b/cmd/routed-eni-cni-plugin/driver/mocks/driver_mocks.go @@ -50,23 +50,23 @@ func (m *MockNetworkAPIs) EXPECT() *MockNetworkAPIsMockRecorder { } // SetupNS mocks base method -func (m *MockNetworkAPIs) SetupNS(arg0, arg1, arg2 string, arg3 *net.IPNet, arg4 int, arg5 []string, arg6 bool, arg7 int, arg8 logger.Logger) error { +func (m *MockNetworkAPIs) SetupNS(arg0, arg1, arg2 string, arg3, arg4 *net.IPNet, arg5 int, arg6 []string, arg7 bool, arg8 int, arg9 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupNS", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) + ret := m.ctrl.Call(m, "SetupNS", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) ret0, _ := ret[0].(error) return ret0 } // SetupNS indicates an expected call of SetupNS -func (mr *MockNetworkAPIsMockRecorder) SetupNS(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { +func (m *MockNetworkAPIs) SetupNS(arg0, arg1, arg2 string, arg3, arg4 *net.IPNet, arg5 int, arg6 []string, arg7 bool, arg8 int, arg9 logger.Logger) error { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupNS", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupNS), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupNS", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupNS), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) } // SetupPodENINetwork mocks base method -func (m *MockNetworkAPIs) SetupPodENINetwork(arg0, arg1, arg2 string, arg3 *net.IPNet, arg4 int, arg5, arg6 string, arg7, arg8 int, arg9 logger.Logger) error { +func (m *MockNetworkAPIs) SetupPodENINetwork(arg0, arg1, arg2 string, arg3, arg4 *net.IPNet, arg5 int, arg6, arg7 string, arg8, arg9 int, arg10 logger.Logger) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetupPodENINetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) + ret := m.ctrl.Call(m, "SetupPodENINetwork", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) ret0, _ := ret[0].(error) return ret0 } @@ -74,7 +74,7 @@ func (m *MockNetworkAPIs) SetupPodENINetwork(arg0, arg1, arg2 string, arg3 *net. // SetupPodENINetwork indicates an expected call of SetupPodENINetwork func (mr *MockNetworkAPIsMockRecorder) SetupPodENINetwork(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodENINetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupPodENINetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupPodENINetwork), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) } // TeardownNS mocks base method diff --git a/misc/10-aws.conflist b/misc/10-aws.conflist index fe26c8346f..fb52e797f4 100644 --- a/misc/10-aws.conflist +++ b/misc/10-aws.conflist @@ -10,6 +10,21 @@ "pluginLogFile": "__PLUGINLOGFILE__", "pluginLogLevel": "__PLUGINLOGLEVEL__" }, + { + "name": "egress-v4-cni", + "type": "egress-v4-cni", + "mtu": 9001, + "enabled": "__EGRESSV4PLUGINENABLED__", + "nodeIP": "__NODEIP__", + "ipam": { + "type": "host-local", + "ranges": [[{"subnet": "169.254.172.0/22"}]], + "routes": [{"dst": "0.0.0.0/0"}], + "dataDir": "/run/cni/v6pd/egress-v4-ipam" + }, + "pluginLogFile": "__EGRESSV4PLUGINLOGFILE__", + "pluginLogLevel": "__PLUGINLOGLEVEL__" + }, { "type": "portmap", "capabilities": {"portMappings": true}, diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 05dea61df4..c4dcf32096 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -120,9 +120,12 @@ type APIs interface { // GetIPv4sFromEC2 returns the IPv4 addresses for a given ENI GetIPv4sFromEC2(eniID string) (addrList []*ec2.NetworkInterfacePrivateIpAddress, err error) - // GetIPv4PrefixesFromEC2 returns the IPv4 addresses for a given ENI + // GetIPv4PrefixesFromEC2 returns the IPv4 prefixes for a given ENI GetIPv4PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv4PrefixSpecification, err error) + // GetIPv6PrefixesFromEC2 returns the IPv6 prefixes for a given ENI + GetIPv6PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv6PrefixSpecification, err error) + // DescribeAllENIs calls EC2 and returns a fully populated DescribeAllENIsResult struct and an error DescribeAllENIs() (DescribeAllENIsResult, error) @@ -138,12 +141,18 @@ type APIs interface { // DeallocPrefixAddresses deallocates the list of IP addresses from a ENI DeallocPrefixAddresses(eniID string, ips []string) error - // GetVPCIPv4CIDRs returns VPC's CIDRs from instance metadata + //AllocIPv6Prefixes allocates IPv6 prefixes to the ENI passed in + AllocIPv6Prefixes(eniID string) ([]*string, error) + + // GetVPCIPv4CIDRs returns VPC's IPv4 CIDRs from instance metadata GetVPCIPv4CIDRs() ([]string, error) - // GetLocalIPv4 returns the primary IP address on the primary ENI interface + // GetLocalIPv4 returns the primary IPv4 address on the primary ENI interface GetLocalIPv4() net.IP + // GetVPCIPv6CIDRs returns VPC's IPv6 CIDRs from instance metadata + GetVPCIPv6CIDRs() ([]string, error) + // GetPrimaryENI returns the primary ENI GetPrimaryENI() string @@ -168,9 +177,12 @@ type APIs interface { //SetCNIunmanaged ENI SetCNIUnmanagedENIs(eniID []string) error - //isCNIUnmanagedENI + //IsCNIUnmanagedENI IsCNIUnmanagedENI(eniID string) bool + //IsPrimaryENI + IsPrimaryENI(eniID string) bool + //RefreshSGIDs RefreshSGIDs(mac string) error @@ -190,6 +202,8 @@ type EC2InstanceMetadataCache struct { securityGroups StringSet subnetID string localIPv4 net.IP + v4Enabled bool + v6Enabled bool instanceID string instanceType string primaryENI string @@ -197,10 +211,10 @@ type EC2InstanceMetadataCache struct { availabilityZone string region string - unmanagedENIs StringSet - useCustomNetworking bool - cniunmanagedENIs StringSet - enableIpv4PrefixDelegation bool + unmanagedENIs StringSet + useCustomNetworking bool + cniunmanagedENIs StringSet + enablePrefixDelegation bool clusterName string additionalENITags map[string]string @@ -220,14 +234,23 @@ type ENIMetadata struct { // DeviceNumber is the device number of network interface DeviceNumber int // 0 means it is primary interface - // SubnetIPv4CIDR is the ipv4 cider of network interface + // SubnetIPv4CIDR is the IPv4 CIDR of network interface SubnetIPv4CIDR string + // SubnetIPv6CIDR is the IPv6 CIDR of network interface + SubnetIPv6CIDR string + // The ip addresses allocated for the network interface IPv4Addresses []*ec2.NetworkInterfacePrivateIpAddress // IPv4 Prefixes allocated for the network interface IPv4Prefixes []*ec2.Ipv4PrefixSpecification + + // IPv6 addresses allocated for the network interface + IPv6Addresses []*ec2.NetworkInterfaceIpv6Address + + // IPv6 Prefixes allocated for the network interface + IPv6Prefixes []*ec2.Ipv6PrefixSpecification } // InstanceTypeLimits keeps track of limits for an instance type @@ -237,7 +260,7 @@ type InstanceTypeLimits struct { HypervisorType string } -// PrimaryIPv4Address returns the primary IP of this node +// PrimaryIPv4Address returns the primary IPv4 address of this node func (eni ENIMetadata) PrimaryIPv4Address() string { for _, addr := range eni.IPv4Addresses { if aws.BoolValue(addr.Primary) { @@ -342,7 +365,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string) } // New creates an EC2InstanceMetadataCache -func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { +func New(useCustomNetworking, v4Enabled, v6Enabled bool) (*EC2InstanceMetadataCache, error) { //ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run ctx := context.Background() @@ -368,6 +391,9 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { cache.useCustomNetworking = useCustomNetworking log.Infof("Custom networking enabled %v", cache.useCustomNetworking) + cache.v4Enabled = v4Enabled + cache.v6Enabled = v6Enabled + awsCfg := aws.NewConfig().WithRegion(region) sess = sess.Copy(awsCfg) @@ -384,9 +410,9 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) { return cache, nil } -func (cache *EC2InstanceMetadataCache) InitCachedPrefixDelegation(enableIpv4PrefixDelegation bool) { - cache.enableIpv4PrefixDelegation = enableIpv4PrefixDelegation - log.Infof("Prefix Delegation enabled %v", cache.enableIpv4PrefixDelegation) +func (cache *EC2InstanceMetadataCache) InitCachedPrefixDelegation(enablePrefixDelegation bool) { + cache.enablePrefixDelegation = enablePrefixDelegation + log.Infof("Prefix Delegation enabled %v", cache.enablePrefixDelegation) } // InitWithEC2metadata initializes the EC2InstanceMetadataCache with the data retrieved from EC2 metadata service @@ -404,7 +430,7 @@ func (cache *EC2InstanceMetadataCache) initWithEC2Metadata(ctx context.Context) if err != nil { return err } - log.Debugf("Discovered the instance primary ip address: %s", cache.localIPv4) + log.Debugf("Discovered the instance primary IPv4 address: %s", cache.localIPv4) // retrieve instance-id cache.instanceID, err = cache.imds.GetInstanceID(ctx) @@ -593,11 +619,24 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat } var ec2ipv4Prefixes []*ec2.Ipv4PrefixSpecification - // Get prefix on primary ENI when custom networking is enabled is not needed. - // If primary ENI has prefixes attached and then we move to custom networking, we don't need to fetch - // the prefix since recommendation is to terminate the nodes and that would have deleted the prefix on the - // primary ENI. - if (eniMAC == primaryMAC && !cache.useCustomNetworking) || (eniMAC != primaryMAC) { + var ec2ipv6Prefixes []*ec2.Ipv6PrefixSpecification + + // If IPv6 is enabled, get attached v6 prefixes. + if cache.v6Enabled { + imdsIPv6Prefixes, err := cache.imds.GetLocalIPv6Prefixes(ctx, eniMAC) + if err != nil { + return ENIMetadata{}, err + } + for _, ipv6prefix := range imdsIPv6Prefixes { + ec2ipv6Prefixes = append(ec2ipv6Prefixes, &ec2.Ipv6PrefixSpecification{ + Ipv6Prefix: aws.String(ipv6prefix.String()), + }) + } + } else if cache.v4Enabled && ((eniMAC == primaryMAC && !cache.useCustomNetworking) || (eniMAC != primaryMAC)) { + // Get prefix on primary ENI when custom networking is enabled is not needed. + // If primary ENI has prefixes attached and then we move to custom networking, we don't need to fetch + // the prefix since recommendation is to terminate the nodes and that would have deleted the prefix on the + // primary ENI. imdsIPv4Prefixes, err := cache.imds.GetLocalIPv4Prefixes(ctx, eniMAC) if err != nil { return ENIMetadata{}, err @@ -616,6 +655,7 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat SubnetIPv4CIDR: cidr.String(), IPv4Addresses: ec2ip4s, IPv4Prefixes: ec2ipv4Prefixes, + IPv6Prefixes: ec2ipv6Prefixes, }, nil } @@ -1021,6 +1061,34 @@ func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (add return returnedENI.Ipv4Prefixes, nil } +// GetIPv6PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI +func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv6PrefixSpecification, err error) { + eniIds := make([]*string, 0) + eniIds = append(eniIds, aws.String(eniID)) + input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} + + start := time.Now() + result, err := cache.ec2SVC.DescribeNetworkInterfacesWithContext(context.Background(), input) + awsAPILatency.WithLabelValues("DescribeNetworkInterfaces", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + if aerr, ok := err.(awserr.Error); ok { + if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" { + return nil, ErrENINotFound + } + } + awsAPIErrInc("DescribeNetworkInterfaces", err) + log.Errorf("Failed to get ENI %s information from EC2 control plane %v", eniID, err) + return nil, errors.Wrap(err, "failed to describe network interface") + } + + if len(result.NetworkInterfaces) == 0 { + return nil, ErrNoNetworkInterfaces + } + returnedENI := result.NetworkInterfaces[0] + + return returnedENI.Ipv6Prefixes, nil +} + // DescribeAllENIs calls EC2 to refresh the ENIMetadata and tags for all attached ENIs func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult, error) { // Fetch all local ENI info from metadata @@ -1320,10 +1388,10 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int } log.Infof("Trying to allocate %d IP addresses on ENI %s", needIPs, eniID) - log.Debugf("PD enabled - %t", cache.enableIpv4PrefixDelegation) + log.Debugf("PD enabled - %t", cache.enablePrefixDelegation) input := &ec2.AssignPrivateIpAddressesInput{} - if cache.enableIpv4PrefixDelegation { + if cache.enablePrefixDelegation { needPrefixes := needIPs input = &ec2.AssignPrivateIpAddressesInput{ NetworkInterfaceId: aws.String(eniID), @@ -1351,7 +1419,7 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int return errors.Wrap(err, "allocate IP/Prefix address: failed to allocate a private IP/Prefix address") } if output != nil { - if cache.enableIpv4PrefixDelegation { + if cache.enablePrefixDelegation { log.Infof("Allocated %d private IP prefixes", len(output.AssignedIpv4Prefixes)) } else { log.Infof("Allocated %d private IP addresses", len(output.AssignedPrivateIpAddresses)) @@ -1360,6 +1428,26 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int return nil } +func (cache *EC2InstanceMetadataCache) AllocIPv6Prefixes(eniID string) ([]*string, error) { + //We only need to allocate one IPv6 prefix per ENI. + input := &ec2.AssignIpv6AddressesInput{ + NetworkInterfaceId: aws.String(eniID), + Ipv6PrefixCount: aws.Int64(1), + } + start := time.Now() + output, err := cache.ec2SVC.AssignIpv6AddressesWithContext(context.Background(), input) + awsAPILatency.WithLabelValues("AssignIpv6AddressesWithContext", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start)) + if err != nil { + log.Errorf("Failed to allocate IPv6 Prefixes on ENI %v: %v", eniID, err) + awsAPIErrInc("AssignPrivateIpv6Addresses", err) + return nil, errors.Wrap(err, "allocate IPv6 prefix: failed to allocate an IPv6 prefix address") + } + if output != nil { + log.Debugf("Allocated %d private IPv6 prefix(es)", len(output.AssignedIpv6Prefixes)) + } + return output.AssignedIpv6Prefixes, nil +} + // WaitForENIAndIPsAttached waits until the ENI has been attached and the secondary IPs have been added func (cache *EC2InstanceMetadataCache) WaitForENIAndIPsAttached(eni string, wantedCidrs int) (eniMetadata ENIMetadata, err error) { return cache.waitForENIAndIPsAttached(eni, wantedCidrs, maxENIBackoffDelay) @@ -1381,8 +1469,13 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want if eni == returnedENI.ENIID { // Check how many Secondary IPs or Prefixes have been attached var eniIPCount int - if cache.enableIpv4PrefixDelegation { + log.Debugf("ENI ID: %v IP Addr: %s, IPv4Prefixes:- %v, IPv6Prefixes:- %v", returnedENI.ENIID, + returnedENI.IPv4Addresses, returnedENI.IPv4Prefixes, returnedENI.IPv6Prefixes) + if cache.enablePrefixDelegation { eniIPCount = len(returnedENI.IPv4Prefixes) + if cache.v6Enabled { + eniIPCount = len(returnedENI.IPv6Prefixes) + } } else { //Ignore primary IP of the ENI //wantedCidrs will be at most 1 less then the IP limit for the ENI because of the primary IP in secondary pod @@ -1410,11 +1503,11 @@ func (cache *EC2InstanceMetadataCache) waitForENIAndIPsAttached(eni string, want if err != nil { // If we have at least 1 Secondary IP, by now return what we have without an error if err == ErrAllSecondaryIPsNotFound { - if !cache.enableIpv4PrefixDelegation && len(eniMetadata.IPv4Addresses) > 1 { + if !cache.enablePrefixDelegation && len(eniMetadata.IPv4Addresses) > 1 { // We have some Secondary IPs, return the ones we have log.Warnf("This ENI only has %d IP addresses, we wanted %d", len(eniMetadata.IPv4Addresses), wantedCidrs) return eniMetadata, nil - } else if cache.enableIpv4PrefixDelegation && len(eniMetadata.IPv4Prefixes) > 1 { + } else if cache.enablePrefixDelegation && len(eniMetadata.IPv4Prefixes) > 1 { // We have some prefixes, return the ones we have log.Warnf("This ENI only has %d Prefixes, we wanted %d", len(eniMetadata.IPv4Prefixes), wantedCidrs) return eniMetadata, nil @@ -1636,6 +1729,23 @@ func (cache *EC2InstanceMetadataCache) GetLocalIPv4() net.IP { return cache.localIPv4 } +// GetVPCIPv6CIDRs returns VPC CIDRs +func (cache *EC2InstanceMetadataCache) GetVPCIPv6CIDRs() ([]string, error) { + ctx := context.TODO() + + ipnets, err := cache.imds.GetVPCIPv6CIDRBlocks(ctx, cache.primaryENImac) + if err != nil { + return nil, err + } + + asStrs := make([]string, len(ipnets)) + for i, ipnet := range ipnets { + asStrs[i] = ipnet.String() + } + + return asStrs, nil +} + // GetPrimaryENI returns the primary ENI func (cache *EC2InstanceMetadataCache) GetPrimaryENI() string { return cache.primaryENI @@ -1697,3 +1807,11 @@ func (cache *EC2InstanceMetadataCache) IsCNIUnmanagedENI(eniID string) bool { } return false } + +//IsPrimaryENI returns if the eni is unmanaged +func (cache *EC2InstanceMetadataCache) IsPrimaryENI(eniID string) bool { + if len(eniID) != 0 && eniID == cache.GetPrimaryENI() { + return true + } + return false +} diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index e0fd9d80e1..b7cd0265a6 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -48,6 +48,7 @@ const ( metadataSubnetCIDR = "/subnet-ipv4-cidr-block" metadataIPv4s = "/local-ipv4s" metadataIPv4Prefixes = "/ipv4-prefix" + metadataIPv6Prefixes = "/ipv6-prefix" az = "us-east-1a" localIP = "10.0.0.10" @@ -69,6 +70,7 @@ const ( eni2Device = "1" eni2PrivateIP = "10.0.0.2" eni2Prefix = "10.0.2.0/28" + eni2v6Prefix = "2001:db8::/64" eni2ID = "eni-12341234" metadataVPCIPv4CIDRs = "192.168.0.0/16 100.66.0.0/1" ) @@ -631,7 +633,7 @@ func TestAllocPrefixAddresses(t *testing.T) { } mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), input, gomock.Any()).Return(nil, nil) - ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enableIpv4PrefixDelegation: true} + ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "c5n.18xlarge", enablePrefixDelegation: true} err := ins.AllocIPAddresses(eniID, 1) assert.NoError(t, err) @@ -648,7 +650,7 @@ func TestAllocPrefixesAlreadyFull(t *testing.T) { NetworkInterfaceId: aws.String(eniID), Ipv4PrefixCount: aws.Int64(1), } - ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "t3.xlarge", enableIpv4PrefixDelegation: true} + ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2, instanceType: "t3.xlarge", enablePrefixDelegation: true} retErr := awserr.New("PrivateIpAddressLimitExceeded", "Too many IPs already allocated", nil) mockEC2.EXPECT().AssignPrivateIpAddressesWithContext(gomock.Any(), input, gomock.Any()).Return(nil, retErr) @@ -760,17 +762,15 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { foundPrefixes int wantedSecondaryIPs int maxBackoffDelay time.Duration + v4Enabled bool + v6Enabled bool times int } - eni1Metadata := ENIMetadata{ - ENIID: eniID, - IPv4Addresses: nil, - IPv4Prefixes: nil, - } + isPrimary := true primaryIP := eni2PrivateIP prefixIP := eni2Prefix - eni2Metadata := ENIMetadata{ + eni1Metadata := ENIMetadata{ ENIID: eni2ID, MAC: eni2MAC, DeviceNumber: 1, @@ -787,15 +787,33 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { }, }, } - eniList := []ENIMetadata{eni1Metadata, eni2Metadata} + v6PrefixIP := eni2v6Prefix + eni2Metadata := ENIMetadata{ + ENIID: eni2ID, + MAC: eni2MAC, + DeviceNumber: 1, + SubnetIPv4CIDR: subnetCIDR, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + Primary: &isPrimary, + PrivateIpAddress: &primaryIP, + }, + }, + IPv6Prefixes: []*ec2.Ipv6PrefixSpecification{ + { + Ipv6Prefix: &v6PrefixIP, + }, + }, + } tests := []struct { name string args args wantEniMetadata ENIMetadata wantErr bool }{ - {"Test wait success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: 1}, eniList[1], false}, - {"Test partial success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries}, eniList[1], false}, + {"Test wait v4 success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: 1, v4Enabled: true, v6Enabled: false}, eni1Metadata, false}, + {"Test partial v4 success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries, v4Enabled: true, v6Enabled: false}, eni1Metadata, false}, + {"Test wait v6 success", args{eni: eni2ID, foundPrefixes: 1, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries, v4Enabled: false, v6Enabled: true}, eni2Metadata, false}, {"Test wait fail", args{eni: eni2ID, foundPrefixes: 0, wantedSecondaryIPs: 1, maxBackoffDelay: 5 * time.Millisecond, times: maxENIEC2APIRetries}, ENIMetadata{}, true}, } for _, tt := range tests { @@ -804,19 +822,24 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { defer ctrl.Finish() eniIPs := eni2PrivateIP eniPrefixes := eni2Prefix + metaDataPrefixPath := metadataIPv4Prefixes + if tt.args.v6Enabled { + eniPrefixes = eni2v6Prefix + metaDataPrefixPath = metadataIPv6Prefixes + } if tt.args.foundPrefixes == 0 { eniPrefixes = "" } - fmt.Println("eniips", eniIPs) mockMetadata := testMetadata(map[string]interface{}{ metadataMACPath: primaryMAC + " " + eni2MAC, metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, metadataMACPath + eni2MAC + metadataInterface: eni2ID, metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, metadataMACPath + eni2MAC + metadataIPv4s: eniIPs, - metadataMACPath + eni2MAC + metadataIPv4Prefixes: eniPrefixes, + metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes, }) - cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, enableIpv4PrefixDelegation: true} + cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, + enablePrefixDelegation: true, v4Enabled: tt.args.v4Enabled, v6Enabled: tt.args.v6Enabled} gotEniMetadata, err := cache.waitForENIAndIPsAttached(tt.args.eni, tt.args.wantedSecondaryIPs, tt.args.maxBackoffDelay) if (err != nil) != tt.wantErr { t.Errorf("waitForENIAndIPsAttached() error = %v, wantErr %v", err, tt.wantErr) @@ -828,6 +851,8 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { }) } } + + func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { mockMetadata := testMetadata(nil) ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go index 91ac4cbade..a26580e678 100644 --- a/pkg/awsutils/imds.go +++ b/pkg/awsutils/imds.go @@ -93,6 +93,11 @@ func (imds TypedIMDS) GetLocalIPv4(ctx context.Context) (net.IP, error) { return imds.getIP(ctx, "local-ipv4") } +// GetLocalIPv6s returns the IPv6 addresses of the instance. +func (imds TypedIMDS) GetLocalIPv6s(ctx context.Context) (net.IP, error) { + return imds.getIP(ctx, "ipv6s") +} + // GetInstanceID returns the ID of this instance. func (imds TypedIMDS) GetInstanceID(ctx context.Context) (string, error) { instanceID, err := imds.GetMetadataWithContext(ctx, "instance-id") @@ -304,6 +309,23 @@ func (imds TypedIMDS) GetLocalIPv4Prefixes(ctx context.Context, mac string) ([]n return prefixes, err } +// GetLocalIPv6Prefixes returns the IPv6 prefixes delegated to this interface +func (imds TypedIMDS) GetLocalIPv6Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/ipv6-prefix", mac) + prefixes, err := imds.getCIDRs(ctx, key) + if err != nil { + if imdsErr, ok := err.(*imdsRequestError); ok { + if IsNotFound(imdsErr.err) { + return nil, nil + } + log.Warnf("%v", err) + return nil, imdsErr.err + } + return nil, err + } + return prefixes, err +} + // GetIPv6s returns the IPv6 addresses associated with the interface. func (imds TypedIMDS) GetIPv6s(ctx context.Context, mac string) ([]net.IP, error) { key := fmt.Sprintf("network/interfaces/macs/%s/ipv6s", mac) @@ -344,7 +366,7 @@ func (imds TypedIMDS) GetVPCIPv4CIDRBlocks(ctx context.Context, mac string) ([]n // GetVPCIPv6CIDRBlocks returns the IPv6 CIDR blocks for the VPC. func (imds TypedIMDS) GetVPCIPv6CIDRBlocks(ctx context.Context, mac string) ([]net.IPNet, error) { - key := fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv6-cidr-blocks", mac) + key := fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-blocks", mac) ipnets, err := imds.getCIDRs(ctx, key) if err != nil { if imdsErr, ok := err.(*imdsRequestError); ok { diff --git a/pkg/awsutils/imds_test.go b/pkg/awsutils/imds_test.go index d120cf1380..409e61aebb 100644 --- a/pkg/awsutils/imds_test.go +++ b/pkg/awsutils/imds_test.go @@ -214,30 +214,14 @@ func TestGetVPCIPv4CIDRBlocks(t *testing.T) { func TestGetVPCIPv6CIDRBlocks(t *testing.T) { f := TypedIMDS{FakeIMDS(map[string]interface{}{ - "network/interfaces/macs/02:c5:f8:3e:6b:27/vpc-ipv6-cidr-blocks": "2001:db8::/64", + "network/interfaces/macs/02:c5:f8:3e:6b:27/subnet-ipv6-cidr-blocks": "2001:db8::/64", })} ips, err := f.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") if assert.NoError(t, err) { - assert.Equal(t, ips, []net.IPNet{{IP: net.ParseIP("2001:db8::"), Mask: net.CIDRMask(64, 128)}}) - } - - nov6 := TypedIMDS{FakeIMDS(map[string]interface{}{ - // NB: IMDS returns 404, not empty string :( - })} - - ips, err = nov6.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") - if imdsErr, ok := err.(*imdsRequestError); ok { - if assert.NoError(t, imdsErr.err) { - assert.ElementsMatch(t, ips, []net.IP{}) - } - } - - _, err = f.GetLocalIPv4s(context.TODO(), "00:00:de:ad:be:ef") - if imdsErr, ok := err.(*imdsRequestError); ok { - if assert.Error(t, imdsErr.err) { - assert.True(t, IsNotFound(imdsErr.err)) - } + assert.Equal(t, ips, + []net.IPNet{{IP: net.IP{0x20, 0x1, 0xd, 0xb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}}) } } diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index f2c69f3f1a..12c4f71008 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -19,12 +19,11 @@ package mock_awsutils import ( - net "net" - reflect "reflect" - awsutils "github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils" ec2 "github.com/aws/aws-sdk-go/service/ec2" gomock "github.com/golang/mock/gomock" + net "net" + reflect "reflect" ) // MockAPIs is a mock of APIs interface @@ -93,6 +92,21 @@ func (mr *MockAPIsMockRecorder) AllocIPAddresses(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPAddresses", reflect.TypeOf((*MockAPIs)(nil).AllocIPAddresses), arg0, arg1) } +// AllocIPv6Prefixes mocks base method +func (m *MockAPIs) AllocIPv6Prefixes(arg0 string) ([]*string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AllocIPv6Prefixes", arg0) + ret0, _ := ret[0].([]*string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AllocIPv6Prefixes indicates an expected call of AllocIPv6Prefixes +func (mr *MockAPIsMockRecorder) AllocIPv6Prefixes(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AllocIPv6Prefixes", reflect.TypeOf((*MockAPIs)(nil).AllocIPv6Prefixes), arg0) +} + // DeallocIPAddresses mocks base method func (m *MockAPIs) DeallocIPAddresses(arg0 string, arg1 []string) error { m.ctrl.T.Helper() @@ -225,6 +239,21 @@ func (mr *MockAPIsMockRecorder) GetIPv4sFromEC2(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv4sFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv4sFromEC2), arg0) } +// GetIPv6PrefixesFromEC2 mocks base method +func (m *MockAPIs) GetIPv6PrefixesFromEC2(arg0 string) ([]*ec2.Ipv6PrefixSpecification, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIPv6PrefixesFromEC2", arg0) + ret0, _ := ret[0].([]*ec2.Ipv6PrefixSpecification) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIPv6PrefixesFromEC2 indicates an expected call of GetIPv6PrefixesFromEC2 +func (mr *MockAPIsMockRecorder) GetIPv6PrefixesFromEC2(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIPv6PrefixesFromEC2", reflect.TypeOf((*MockAPIs)(nil).GetIPv6PrefixesFromEC2), arg0) +} + // GetInstanceHypervisorFamily mocks base method func (m *MockAPIs) GetInstanceHypervisorFamily() (string, error) { m.ctrl.T.Helper() @@ -311,6 +340,21 @@ func (mr *MockAPIsMockRecorder) GetVPCIPv4CIDRs() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVPCIPv4CIDRs", reflect.TypeOf((*MockAPIs)(nil).GetVPCIPv4CIDRs)) } +// GetVPCIPv6CIDRs mocks base method +func (m *MockAPIs) GetVPCIPv6CIDRs() ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetVPCIPv6CIDRs") + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetVPCIPv6CIDRs indicates an expected call of GetVPCIPv6CIDRs +func (mr *MockAPIsMockRecorder) GetVPCIPv6CIDRs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVPCIPv6CIDRs", reflect.TypeOf((*MockAPIs)(nil).GetVPCIPv6CIDRs)) +} + // InitCachedPrefixDelegation mocks base method func (m *MockAPIs) InitCachedPrefixDelegation(arg0 bool) { m.ctrl.T.Helper() @@ -337,6 +381,20 @@ func (mr *MockAPIsMockRecorder) IsCNIUnmanagedENI(arg0 interface{}) *gomock.Call return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCNIUnmanagedENI", reflect.TypeOf((*MockAPIs)(nil).IsCNIUnmanagedENI), arg0) } +// IsPrimaryENI mocks base method +func (m *MockAPIs) IsPrimaryENI(arg0 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsPrimaryENI", arg0) + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsPrimaryENI indicates an expected call of IsPrimaryENI +func (mr *MockAPIsMockRecorder) IsPrimaryENI(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsPrimaryENI", reflect.TypeOf((*MockAPIs)(nil).IsPrimaryENI), arg0) +} + // IsUnmanagedENI mocks base method func (m *MockAPIs) IsUnmanagedENI(arg0 string) bool { m.ctrl.T.Helper() diff --git a/pkg/ec2wrapper/client.go b/pkg/ec2wrapper/client.go index 77dd743e74..e88ea29c63 100644 --- a/pkg/ec2wrapper/client.go +++ b/pkg/ec2wrapper/client.go @@ -30,6 +30,8 @@ type EC2 interface { DetachNetworkInterfaceWithContext(ctx aws.Context, input *ec2svc.DetachNetworkInterfaceInput, opts ...request.Option) (*ec2svc.DetachNetworkInterfaceOutput, error) AssignPrivateIpAddressesWithContext(ctx aws.Context, input *ec2svc.AssignPrivateIpAddressesInput, opts ...request.Option) (*ec2svc.AssignPrivateIpAddressesOutput, error) UnassignPrivateIpAddressesWithContext(ctx aws.Context, input *ec2svc.UnassignPrivateIpAddressesInput, opts ...request.Option) (*ec2svc.UnassignPrivateIpAddressesOutput, error) + AssignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.AssignIpv6AddressesInput, opts ...request.Option) (*ec2svc.AssignIpv6AddressesOutput, error) + UnassignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.UnassignIpv6AddressesInput, opts ...request.Option) (*ec2svc.UnassignIpv6AddressesOutput, error) DescribeNetworkInterfacesWithContext(ctx aws.Context, input *ec2svc.DescribeNetworkInterfacesInput, opts ...request.Option) (*ec2svc.DescribeNetworkInterfacesOutput, error) ModifyNetworkInterfaceAttributeWithContext(ctx aws.Context, input *ec2svc.ModifyNetworkInterfaceAttributeInput, opts ...request.Option) (*ec2svc.ModifyNetworkInterfaceAttributeOutput, error) CreateTagsWithContext(ctx aws.Context, input *ec2svc.CreateTagsInput, opts ...request.Option) (*ec2svc.CreateTagsOutput, error) diff --git a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go index f4791e0abd..9613a99eb7 100644 --- a/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go +++ b/pkg/ec2wrapper/mocks/ec2wrapper_mocks.go @@ -63,13 +63,33 @@ func (m *MockEC2) AssignPrivateIpAddressesWithContext(arg0 context.Context, arg1 return ret0, ret1 } -// AssignPrivateIpAddressesWithContext indicates an expected call of AssignPrivateIpAddressesWithContext +// AssignPrivateIpAddressesWithContext indicates an expected call of AssignIpv6AddressesWithContext func (mr *MockEC2MockRecorder) AssignPrivateIpAddressesWithContext(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AssignPrivateIpAddressesWithContext", reflect.TypeOf((*MockEC2)(nil).AssignPrivateIpAddressesWithContext), varargs...) } +// AssignIpv6AddressesWithContext mocks base method +func (m *MockEC2) AssignIpv6AddressesWithContext(arg0 context.Context, arg1 *ec2.AssignIpv6AddressesInput, arg2 ...request.Option) (*ec2.AssignIpv6AddressesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AssignIpv6AddressesWithContext", varargs...) + ret0, _ := ret[0].(*ec2.AssignIpv6AddressesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AssignIpv6AddressesWithContext indicates an expected call of AssignIpv6AddressesWithContext +func (mr *MockEC2MockRecorder) AssignIpv6AddressesWithContext(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AssignIpv6AddressesWithContext", reflect.TypeOf((*MockEC2)(nil).AssignIpv6AddressesWithContext), varargs...) +} + // AttachNetworkInterfaceWithContext mocks base method func (m *MockEC2) AttachNetworkInterfaceWithContext(arg0 context.Context, arg1 *ec2.AttachNetworkInterfaceInput, arg2 ...request.Option) (*ec2.AttachNetworkInterfaceOutput, error) { m.ctrl.T.Helper() @@ -288,3 +308,23 @@ func (mr *MockEC2MockRecorder) UnassignPrivateIpAddressesWithContext(arg0, arg1 varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnassignPrivateIpAddressesWithContext", reflect.TypeOf((*MockEC2)(nil).UnassignPrivateIpAddressesWithContext), varargs...) } + +// UnassignIpv6AddressesWithContext mocks base method +func (m *MockEC2) UnassignIpv6AddressesWithContext(arg0 context.Context, arg1 *ec2.UnassignIpv6AddressesInput, arg2 ...request.Option) (*ec2.UnassignIpv6AddressesOutput, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "UnassignIpv6AddressesWithContext", varargs...) + ret0, _ := ret[0].(*ec2.UnassignIpv6AddressesOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UnassignIpv6AddressesWithContext indicates an expected call of UnassignPrivateIpAddressesWithContext +func (mr *MockEC2MockRecorder) UnassignIpv6AddressesWithContext(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnassignPrivateIpAddressesWithContext", reflect.TypeOf((*MockEC2)(nil).UnassignIpv6AddressesWithContext), varargs...) +} diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index e34ef0b050..2c10f3a288 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -78,6 +78,8 @@ const checkpointMigrationPhase = 1 const backfillNetworkName = "_migrated-from-cri" const backfillNetworkIface = "unknown" +const defaultMaxIPv6addresses = 250 + // ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, sandbox id var ErrUnknownPod = errors.New("datastore: unknown pod") @@ -164,6 +166,8 @@ type ENI struct { // be in dot-decimal notation with no leading zeros and no whitespace(eg: "10.1.0.253") // Key is the IP address - PD: "IP/28" and SIP: "IP/32" AvailableIPv4Cidrs map[string]*CidrInfo + //IPv6CIDRs contains information tied to IPv6 Prefixes attached to the ENI + IPv6Cidrs map[string]*CidrInfo } // AddressInfo contains information about an IP, Exported fields will be marshaled for introspection. @@ -175,12 +179,14 @@ type AddressInfo struct { // CidrInfo type CidrInfo struct { - //Cidr info /32 or /28 prefix + //Either v4/v6 Host or LPM Prefix Cidr net.IPNet - //Key is the /32 IP either secondary IP or free /32 IP allocated from /28 prefix - IPv4Addresses map[string]*AddressInfo - //This block of addresses was allocated through PrefixDelegation + //Key is individual IP addresses from the Prefix - /32 (v4) or /128 (v6) + IPAddresses map[string]*AddressInfo + //true if Cidr here is an LPM prefix IsPrefix bool + //IP Address Family of the Cidr + AddressFamily string } func (cidr *CidrInfo) Size() int { @@ -189,8 +195,19 @@ func (cidr *CidrInfo) Size() int { } func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { + //Either v4 or v6 for now. + //Check in V4 prefixes for _, availableCidr := range e.AvailableIPv4Cidrs { - for _, addr := range availableCidr.IPv4Addresses { + for _, addr := range availableCidr.IPAddresses { + if addr.IPAMKey == ipamKey { + return availableCidr, addr + } + } + } + + //Check in V6 prefixes + for _, availableCidr := range e.IPv6Cidrs { + for _, addr := range availableCidr.IPAddresses { if addr.IPAMKey == ipamKey { return availableCidr, addr } @@ -203,17 +220,17 @@ func (e *ENI) findAddressForSandbox(ipamKey IPAMKey) (*CidrInfo, *AddressInfo) { func (e *ENI) AssignedIPv4Addresses() int { count := 0 for _, availableCidr := range e.AvailableIPv4Cidrs { - count += availableCidr.AssignedIPv4AddressesInCidr() + count += availableCidr.AssignedIPAddressesInCidr() } return count } -//AssignedIPv4AddressesInCidr is the number of IP addresses already assigned in the CIDR -func (cidr *CidrInfo) AssignedIPv4AddressesInCidr() int { +//AssignedIPAddressesInCidr is the number of IP addresses already assigned in the IPv4 CIDR +func (cidr *CidrInfo) AssignedIPAddressesInCidr() int { count := 0 //SIP : This will run just once and count will be 0 if addr is not assigned or addr is not allocated yet(unused IP) //PD : This will return count of number /32 assigned in /28 CIDR. - for _, addr := range cidr.IPv4Addresses { + for _, addr := range cidr.IPAddresses { if addr.Assigned() { count++ } @@ -327,13 +344,13 @@ type CheckpointData struct { // in checkpoints. type CheckpointEntry struct { IPAMKey - IPv4 string `json:"ipv4"` + IP string `json:"ip"` } // ReadBackingStore initialises the IP allocation state from the // configured backing store. Should be called before using data // store. -func (ds *DataStore) ReadBackingStore() error { +func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { var data CheckpointData switch ds.CheckpointMigrationPhase { @@ -348,14 +365,15 @@ func (ds *DataStore) ReadBackingStore() error { entries := make([]CheckpointEntry, 0, len(sandboxes)) for _, s := range sandboxes { + ds.log.Debugf("Adding container ID: %v", s.ID) entries = append(entries, CheckpointEntry{ - // NB: These Backfill values are also assumed in UnassignPodIPv4Address + // NB: These Backfill values are also assumed in UnassignPodIPAddress IPAMKey: IPAMKey{ NetworkName: backfillNetworkName, ContainerID: s.ID, IfName: backfillNetworkIface, }, - IPv4: s.IP, + IP: s.IP, }) } data = CheckpointData{ @@ -391,20 +409,26 @@ func (ds *DataStore) ReadBackingStore() error { defer ds.lock.Unlock() for _, allocation := range data.Allocations { - ipv4Addr := net.ParseIP(allocation.IPv4) + ipAddr := net.ParseIP(allocation.IP) found := false eniloop: for _, eni := range ds.eniPool { - for _, cidr := range eni.AvailableIPv4Cidrs { - if cidr.Cidr.Contains(ipv4Addr) { + eniCidrs := eni.AvailableIPv4Cidrs + if isv6Enabled { + ds.log.Debugf("v6 is enabled") + eniCidrs = eni.IPv6Cidrs + } + for _, cidr := range eniCidrs { + ds.log.Debugf("Checking if IP: %v belongs to CIDR: %v", ipAddr, cidr.Cidr) + if cidr.Cidr.Contains(ipAddr) { // Found! found = true - if _, ok := cidr.IPv4Addresses[allocation.IPv4]; ok { + if _, ok := cidr.IPAddresses[allocation.IP]; ok { return errors.New(IPAlreadyInStoreError) } - addr := &AddressInfo{Address: ipv4Addr.String()} - cidr.IPv4Addresses[allocation.IPv4] = addr - ds.assignPodIPv4AddressUnsafe(allocation.IPAMKey, eni, addr) + addr := &AddressInfo{Address: ipAddr.String()} + cidr.IPAddresses[allocation.IP] = addr + ds.assignPodIPAddressUnsafe(allocation.IPAMKey, eni, addr) ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) //Update prometheus for ips per cidr //Secondary IP mode will have /32:1 and Prefix mode will have /28: @@ -414,7 +438,7 @@ func (ds *DataStore) ReadBackingStore() error { } } if !found { - ds.log.Infof("datastore: Sandbox %s uses unknown IPv4 %s - presuming stale/dead", allocation.IPAMKey, allocation.IPv4) + ds.log.Infof("datastore: Sandbox %s uses unknown IP Address %s - presuming stale/dead", allocation.IPAMKey, allocation.IP) } } @@ -436,12 +460,25 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { allocations := make([]CheckpointEntry, 0, ds.assigned) for _, eni := range ds.eniPool { + //Loop through ENI's v4 prefixes for _, assignedAddr := range eni.AvailableIPv4Cidrs { - for _, addr := range assignedAddr.IPv4Addresses { + for _, addr := range assignedAddr.IPAddresses { + if addr.Assigned() { + entry := CheckpointEntry{ + IPAMKey: addr.IPAMKey, + IP: addr.Address, + } + allocations = append(allocations, entry) + } + } + } + //Loop through ENI's v6 prefixes + for _, assignedAddr := range eni.IPv6Cidrs { + for _, addr := range assignedAddr.IPAddresses { if addr.Assigned() { entry := CheckpointEntry{ IPAMKey: addr.IPAMKey, - IPv4: addr.Address, + IP: addr.Address, } allocations = append(allocations, entry) } @@ -481,7 +518,7 @@ func (ds *DataStore) AddENI(eniID string, deviceNumber int, isPrimary, isTrunk, return nil } -// AddIPv4AddressToStore add CIDR of an ENI to data store +// AddIPv4AddressToStore adds IPv4 CIDR of an ENI to data store func (ds *DataStore) AddIPv4CidrToStore(eniID string, ipv4Cidr net.IPNet, isPrefix bool) error { ds.lock.Lock() defer ds.lock.Unlock() @@ -502,8 +539,9 @@ func (ds *DataStore) AddIPv4CidrToStore(eniID string, ipv4Cidr net.IPNet, isPref newCidrInfo := &CidrInfo{ Cidr: ipv4Cidr, - IPv4Addresses: make(map[string]*AddressInfo), + IPAddresses: make(map[string]*AddressInfo), IsPrefix: isPrefix, + AddressFamily: "4", } curENI.AvailableIPv4Cidrs[strIPv4Cidr] = newCidrInfo @@ -541,13 +579,13 @@ func (ds *DataStore) DelIPv4CidrFromStore(eniID string, cidr net.IPNet, force bo // PD case : if (force is false) then if there are any unassigned IPs, those will get freed but the first assigned IP will // break the loop, should be fine since freed IPs will be reused for new pods. updateBackingStore := false - for _, addr := range deletableCidr.IPv4Addresses { + for _, addr := range deletableCidr.IPAddresses { if addr.Assigned() { if !force { return errors.New(IPInUseError) } forceRemovedIPs.Inc() - ds.unassignPodIPv4AddressUnsafe(addr) + ds.unassignPodIPAddressUnsafe(addr) updateBackingStore = true } } @@ -569,6 +607,111 @@ func (ds *DataStore) DelIPv4CidrFromStore(eniID string, cidr net.IPNet, force bo return nil } +// AddIPv6AddressToStore adds IPv6 CIDR of an ENI to data store +func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPrefix bool) error { + ds.lock.Lock() + defer ds.lock.Unlock() + + strIPv6Cidr := ipv6Cidr.String() + ds.log.Debugf("Adding %s to DS for %s", strIPv6Cidr, eniID) + curENI, ok := ds.eniPool[eniID] + ds.log.Debugf("ENI in pool %s", ok) + + if !ok { + ds.log.Debugf("unkown ENI") + return errors.New("add ENI's IP to datastore: unknown ENI") + } + // Check if already present in datastore. + _, ok = curENI.IPv6Cidrs[strIPv6Cidr] + ds.log.Debugf("IP not in DS") + if ok { + ds.log.Debugf("IPv6 prefix %s already in DS", strIPv6Cidr) + return errors.New(IPAlreadyInStoreError) + } + + ds.log.Debugf("Assigning IPv6CIDRs") + if curENI.IPv6Cidrs == nil { curENI.IPv6Cidrs = make(map[string]*CidrInfo) } + curENI.IPv6Cidrs[strIPv6Cidr] = &CidrInfo{ + Cidr: ipv6Cidr, + IPAddresses: make(map[string]*AddressInfo), + IsPrefix: isPrefix, + AddressFamily: "6", + } + + //curENI.IPv6Cidrs[strIPv6Cidr].Size() will end up with a huge number. So, instead capping it to Max Pods ceiling on + //an EKS Node. TODO - Should we increase the default value further as VPC CNI can also be used in a Self Managed K8S + //cluster with no such upper bound? + ds.total += defaultMaxIPv6addresses + if isPrefix { + ds.allocatedPrefix++ + } + totalIPs.Set(float64(ds.total)) + + ds.log.Debugf("Added ENI(%s)'s IP/Prefix %s to datastore", eniID, strIPv6Cidr) + return nil +} + +func (ds *DataStore) AssignPodIPAddress(ipamKey IPAMKey, isIPv4Enabled bool, isIPv6Enabled bool) (ipv4Address string, + ipv6Address string, deviceNumber int, err error) { + //Currently it's either v4 or v6. Dual Stack mode isn't supported. + if isIPv4Enabled { + ipv4Address, deviceNumber, err = ds.AssignPodIPv4Address(ipamKey) + } else if isIPv6Enabled { + ipv6Address, deviceNumber, err = ds.AssignPodIPv6Address(ipamKey) + } + return ipv4Address, ipv6Address, deviceNumber, err +} + +// AssignPodIPv6Address assigns an IPv6 address to pod. Returns the assigned IPv6 address along with device number +func (ds *DataStore) AssignPodIPv6Address(ipamKey IPAMKey) (ipv6Address string, deviceNumber int, err error) { + ds.lock.Lock() + defer ds.lock.Unlock() + + if !ds.isPDEnabled { + return "", -1, fmt.Errorf("PD is not enabled. V6 is only supported in PD mode") + } + ds.log.Debugf("AssignIPv6Address: IPv6 address pool stats: assigned %d", ds.assigned) + + if eni, _, addr := ds.eniPool.FindAddressForSandbox(ipamKey); addr != nil { + ds.log.Infof("AssignPodIPv6Address: duplicate pod assign for sandbox %s", ipamKey) + return addr.Address, eni.DeviceNumber, nil + } + + //In IPv6 Prefix Delegation mode, eniPool will only have Primary ENI. + for _, eni := range ds.eniPool { + if len(eni.IPv6Cidrs) == 0 { + continue + } + for _, V6Cidr := range eni.IPv6Cidrs { + if !V6Cidr.IsPrefix { + continue + } + ipv6Address, err = ds.getFreeIPv6AddrFromCidr(V6Cidr) + if err != nil { + ds.log.Debugf("Unable to get IP address from prefix: %v", err) + //In v6 mode, we (should) only have one CIDR/Prefix. So, we can bail out but we will let the loop + //exit instead. + continue + } + ds.log.Debugf("New v6 IP from PD pool- %s", ipv6Address) + addr := &AddressInfo{Address: ipv6Address} + V6Cidr.IPAddresses[ipv6Address] = addr + + ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) + if err := ds.writeBackingStoreUnsafe(); err != nil { + ds.log.Warnf("Failed to update backing store: %v", err) + // Important! Unwind assignment + ds.unassignPodIPAddressUnsafe(addr) + //Remove the IP from eni DB + delete(V6Cidr.IPAddresses, addr.Address) + return "", -1, err + } + return addr.Address, eni.DeviceNumber, nil + } + } + return "", -1, errors.New("assignPodIPv6AddressUnsafe: no available IP addresses") +} + // AssignPodIPv4Address assigns an IPv4 address to pod // It returns the assigned IPv4 address, device number, error func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, deviceNumber int, err error) { @@ -596,8 +739,8 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, continue } ds.log.Debugf("New IP from CIDR pool- %s", strPrivateIPv4) - if availableCidr.IPv4Addresses == nil { - availableCidr.IPv4Addresses = make(map[string]*AddressInfo) + if availableCidr.IPAddresses == nil { + availableCidr.IPAddresses = make(map[string]*AddressInfo) } //Update prometheus for ips per cidr //Secondary IP mode will have /32:1 and Prefix mode will have /28: @@ -608,22 +751,22 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, continue } - addr = availableCidr.IPv4Addresses[strPrivateIPv4] + addr = availableCidr.IPAddresses[strPrivateIPv4] if addr == nil { // addr is nil when we are using a new IP from prefix or SIP pool // if addr is out of cooldown or not assigned, we can reuse addr addr = &AddressInfo{Address: strPrivateIPv4} } - availableCidr.IPv4Addresses[strPrivateIPv4] = addr - ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + availableCidr.IPAddresses[strPrivateIPv4] = addr + ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) if err := ds.writeBackingStoreUnsafe(); err != nil { ds.log.Warnf("Failed to update backing store: %v", err) // Important! Unwind assignment - ds.unassignPodIPv4AddressUnsafe(addr) + ds.unassignPodIPAddressUnsafe(addr) //Remove the IP from eni DB - delete(availableCidr.IPv4Addresses, addr.Address) + delete(availableCidr.IPAddresses, addr.Address) //Update prometheus for ips per cidr ipsPerCidr.With(prometheus.Labels{"cidr": availableCidr.Cidr.String()}).Dec() return "", -1, err @@ -638,7 +781,7 @@ func (ds *DataStore) AssignPodIPv4Address(ipamKey IPAMKey) (ipv4address string, } // It returns the assigned IPv4 address, device number -func (ds *DataStore) assignPodIPv4AddressUnsafe(ipamKey IPAMKey, eni *ENI, addr *AddressInfo) (string, int) { +func (ds *DataStore) assignPodIPAddressUnsafe(ipamKey IPAMKey, eni *ENI, addr *AddressInfo) (string, int) { ds.log.Infof("AssignPodIPv4Address: Assign IP %v to sandbox %s", addr.Address, ipamKey) @@ -654,12 +797,12 @@ func (ds *DataStore) assignPodIPv4AddressUnsafe(ipamKey IPAMKey, eni *ENI, addr return addr.Address, eni.DeviceNumber } -func (ds *DataStore) unassignPodIPv4AddressUnsafe(addr *AddressInfo) { +func (ds *DataStore) unassignPodIPAddressUnsafe(addr *AddressInfo) { if !addr.Assigned() { // Already unassigned return } - ds.log.Infof("UnAssignPodIPv4Address: Unassign IP %v from sandbox %s", + ds.log.Infof("UnAssignPodIPAddress: Unassign IP %v from sandbox %s", addr.Address, addr.IPAMKey) addr.IPAMKey = IPAMKey{} // unassign the addr ds.assigned-- @@ -668,20 +811,28 @@ func (ds *DataStore) unassignPodIPv4AddressUnsafe(addr *AddressInfo) { } // GetStats returns total number of IP addresses, number of assigned IP addresses and total prefixes -func (ds *DataStore) GetStats() (int, int, int) { +func (ds *DataStore) GetStats(addressFamily string) (int, int, int) { ds.lock.Lock() defer ds.lock.Unlock() totalIPs := 0 assignedIPs := 0 for _, eni := range ds.eniPool { - for _, cidr := range eni.AvailableIPv4Cidrs { - if (ds.isPDEnabled && cidr.IsPrefix) || (!ds.isPDEnabled && !cidr.IsPrefix) { - assignedIPs += cidr.AssignedIPv4AddressesInCidr() + AssignedCIDRs := eni.AvailableIPv4Cidrs + if addressFamily == "6" { + AssignedCIDRs = eni.IPv6Cidrs + } + for _, cidr := range AssignedCIDRs { + if addressFamily == "4" && ((ds.isPDEnabled && cidr.IsPrefix) || (!ds.isPDEnabled && !cidr.IsPrefix)) { + assignedIPs += cidr.AssignedIPAddressesInCidr() totalIPs += cidr.Size() + } else if addressFamily == "6" { + assignedIPs += cidr.AssignedIPAddressesInCidr() + //Set to default Max pods we support on an instance, otherwise we will end up displaying + //a huge number. + totalIPs += defaultMaxIPv6addresses } } - } return totalIPs, assignedIPs, ds.allocatedPrefix } @@ -719,7 +870,7 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENI) bool if other.ID != eni.ID { for _, otherPrefixes := range other.AvailableIPv4Cidrs { if (ds.isPDEnabled && otherPrefixes.IsPrefix == true) || (!ds.isPDEnabled && otherPrefixes.IsPrefix == false) { - otherWarmIPs += otherPrefixes.Size() - otherPrefixes.AssignedIPv4AddressesInCidr() + otherWarmIPs += otherPrefixes.Size() - otherPrefixes.AssignedIPAddressesInCidr() } } } @@ -762,7 +913,7 @@ func (ds *DataStore) isRequiredForWarmPrefixTarget(warmPrefixTarget int, eni *EN for _, other := range ds.eniPool { if other.ID != eni.ID { for _, otherPrefixes := range other.AvailableIPv4Cidrs { - if otherPrefixes.AssignedIPv4AddressesInCidr() == 0 { + if otherPrefixes.AssignedIPAddressesInCidr() == 0 { freePrefixes++ } } @@ -832,7 +983,7 @@ func (e *ENI) isTooYoung() bool { // HasIPInCooling returns true if an IP address was unassigned recently. func (e *ENI) hasIPInCooling() bool { for _, assignedaddr := range e.AvailableIPv4Cidrs { - for _, addr := range assignedaddr.IPv4Addresses { + for _, addr := range assignedaddr.IPAddresses { if addr.inCoolingPeriod() { return true } @@ -917,9 +1068,9 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { forceRemovedENIs.Inc() forceRemovedIPs.Add(float64(eni.AssignedIPv4Addresses())) for _, assignedaddr := range eni.AvailableIPv4Cidrs { - for _, addr := range assignedaddr.IPv4Addresses { + for _, addr := range assignedaddr.IPAddresses { if addr.Assigned() { - ds.unassignPodIPv4AddressUnsafe(addr) + ds.unassignPodIPAddressUnsafe(addr) } } ds.total -= assignedaddr.Size() @@ -949,12 +1100,12 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { return nil } -// UnassignPodIPv4Address a) find out the IP address based on PodName and PodNameSpace +// UnassignPodIPAddress a) find out the IP address based on PodName and PodNameSpace // b) mark IP address as unassigned c) returns IP address, ENI's device number, error -func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) { +func (ds *DataStore) UnassignPodIPAddress(ipamKey IPAMKey) (e *ENI, ip string, deviceNumber int, err error) { ds.lock.Lock() defer ds.lock.Unlock() - ds.log.Debugf("UnassignPodIPv4Address: IP address pool stats: total:%d, assigned %d, sandbox %s", + ds.log.Debugf("UnassignPodIPAddress: IP address pool stats: total:%d, assigned %d, sandbox %s", ds.total, ds.assigned, ipamKey) eni, availableCidr, addr := ds.eniPool.FindAddressForSandbox(ipamKey) @@ -968,21 +1119,22 @@ func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (e *ENI, ip string, // restart rather than by observing an ADD operation // directly, then we won't have captured the true // networkname/ifname. - ds.log.Debugf("UnassignPodIPv4Address: Failed to find IPAM entry under full key, trying CRI-migrated version") + ds.log.Debugf("UnassignPodIPAddress: Failed to find IPAM entry under full key, trying CRI-migrated version") ipamKey.NetworkName = backfillNetworkName ipamKey.IfName = backfillNetworkIface eni, availableCidr, addr = ds.eniPool.FindAddressForSandbox(ipamKey) } if addr == nil { - ds.log.Warnf("UnassignPodIPv4Address: Failed to find sandbox %s", + ds.log.Warnf("UnassignPodIPAddress: Failed to find sandbox %s", ipamKey) + //Pod Not found. Nothing to do from IPAMD perspective. return nil, "", 0, ErrUnknownPod } - ds.unassignPodIPv4AddressUnsafe(addr) + ds.unassignPodIPAddressUnsafe(addr) if err := ds.writeBackingStoreUnsafe(); err != nil { // Unwind un-assignment - ds.assignPodIPv4AddressUnsafe(ipamKey, eni, addr) + ds.assignPodIPAddressUnsafe(ipamKey, eni, addr) return nil, "", 0, err } addr.UnassignedTime = time.Now() @@ -992,8 +1144,7 @@ func (ds *DataStore) UnassignPodIPv4Address(ipamKey IPAMKey) (e *ENI, ip string, } //Update prometheus for ips per cidr ipsPerCidr.With(prometheus.Labels{"cidr": availableCidr.Cidr.String()}).Dec() - - ds.log.Infof("UnassignPodIPv4Address: sandbox %s's ipAddr %s, DeviceNumber %d", + ds.log.Infof("UnassignPodIPAddress: sandbox %s's ipAddr %s, DeviceNumber %d", ipamKey, addr.Address, eni.DeviceNumber) return eni, addr.Address, eni.DeviceNumber, nil } @@ -1007,7 +1158,7 @@ func (ds *DataStore) AllocatedIPs() []PodIPInfo { ret := make([]PodIPInfo, 0, ds.eniPool.AssignedIPv4Addresses()) for _, eni := range ds.eniPool { for _, assignedaddr := range eni.AvailableIPv4Cidrs { - for _, addr := range assignedaddr.IPv4Addresses { + for _, addr := range assignedaddr.IPAddresses { if addr.Assigned() { info := PodIPInfo{ IPAMKey: addr.IPAMKey, @@ -1036,7 +1187,7 @@ func (ds *DataStore) FreeableIPs(eniID string) []net.IPNet { freeable := make([]net.IPNet, 0, len(eni.AvailableIPv4Cidrs)) for _, assignedaddr := range eni.AvailableIPv4Cidrs { - if !assignedaddr.IsPrefix && assignedaddr.AssignedIPv4AddressesInCidr() == 0 { + if !assignedaddr.IsPrefix && assignedaddr.AssignedIPAddressesInCidr() == 0 { freeable = append(freeable, assignedaddr.Cidr) } } @@ -1058,7 +1209,7 @@ func (ds *DataStore) FreeablePrefixes(eniID string) []net.IPNet { freeable := make([]net.IPNet, 0, len(eni.AvailableIPv4Cidrs)) for _, assignedaddr := range eni.AvailableIPv4Cidrs { - if assignedaddr.IsPrefix && assignedaddr.AssignedIPv4AddressesInCidr() == 0 { + if assignedaddr.IsPrefix && assignedaddr.AssignedIPAddressesInCidr() == 0 { freeable = append(freeable, assignedaddr.Cidr) } } @@ -1079,16 +1230,29 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { for eni, eniInfo := range ds.eniPool { tmpENIInfo := *eniInfo tmpENIInfo.AvailableIPv4Cidrs = make(map[string]*CidrInfo, len(eniInfo.AvailableIPv4Cidrs)) + tmpENIInfo.IPv6Cidrs = make(map[string]*CidrInfo, len(eniInfo.IPv6Cidrs)) for cidr, _ := range eniInfo.AvailableIPv4Cidrs { tmpENIInfo.AvailableIPv4Cidrs[cidr] = &CidrInfo{ Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr, - IPv4Addresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPv4Addresses)), + IPAddresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)), IsPrefix: eniInfo.AvailableIPv4Cidrs[cidr].IsPrefix, } // Since IP Addresses might get removed, we need to make a deep copy here. - for ip, ipAddrInfoRef := range eniInfo.AvailableIPv4Cidrs[cidr].IPv4Addresses { + for ip, ipAddrInfoRef := range eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses { ipAddrInfo := *ipAddrInfoRef - tmpENIInfo.AvailableIPv4Cidrs[cidr].IPv4Addresses[ip] = &ipAddrInfo + tmpENIInfo.AvailableIPv4Cidrs[cidr].IPAddresses[ip] = &ipAddrInfo + } + } + for cidr, _ := range eniInfo.IPv6Cidrs { + tmpENIInfo.IPv6Cidrs[cidr] = &CidrInfo{ + Cidr: eniInfo.IPv6Cidrs[cidr].Cidr, + IPAddresses: make(map[string]*AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)), + IsPrefix: eniInfo.IPv6Cidrs[cidr].IsPrefix, + } + // Since IP Addresses might get removed, we need to make a deep copy here. + for ip, ipAddrInfoRef := range eniInfo.IPv6Cidrs[cidr].IPAddresses { + ipAddrInfo := *ipAddrInfoRef + tmpENIInfo.IPv6Cidrs[cidr].IPAddresses[ip] = &ipAddrInfo } } eniInfos.ENIs[eni] = tmpENIInfo @@ -1133,7 +1297,7 @@ func (ds *DataStore) GetFreePrefixes() int { freePrefixes := 0 for _, other := range ds.eniPool { for _, otherPrefixes := range other.AvailableIPv4Cidrs { - if otherPrefixes.IsPrefix && otherPrefixes.AssignedIPv4AddressesInCidr() == 0 { + if otherPrefixes.IsPrefix && otherPrefixes.AssignedIPAddressesInCidr() == 0 { freePrefixes++ } } @@ -1157,6 +1321,20 @@ func (ds *DataStore) getFreeIPv4AddrfromCidr(availableCidr *CidrInfo) (string, e return strPrivateIPv4, nil } +func (ds *DataStore) getFreeIPv6AddrFromCidr(IPv6Cidr *CidrInfo) (string, error) { + if IPv6Cidr == nil { + ds.log.Errorf("Prefix datastore not initialized") + return "", errors.New("Prefix datastore not initialized") + } + ipv6Address, err := ds.getUnusedIP(IPv6Cidr) + if err != nil { + ds.log.Debugf("Get free IP from prefix failed %v", err) + return "", err + } + ds.log.Debugf("Returning Free IP %s", ipv6Address) + return ipv6Address, nil +} + /* /28 -> 10.1.1.1/32[out of cooldown], 10.1.1.2/32[out of cooldown], 10.1.1.3/32[assigned] cached ip = 10.1.1.1/32 @@ -1167,14 +1345,15 @@ func (ds *DataStore) getFreeIPv4AddrfromCidr(availableCidr *CidrInfo) (string, e func (ds *DataStore) getUnusedIP(availableCidr *CidrInfo) (string, error) { //Check if there is any IP out of cooldown var cachedIP string - for _, addr := range availableCidr.IPv4Addresses { + for _, addr := range availableCidr.IPAddresses { if !addr.Assigned() && !addr.inCoolingPeriod() { //if the IP is out of cooldown and not assigned then cache the first available IP //continue cleaning up the DB, this is to avoid stale entries and a new thread :) if cachedIP == "" { cachedIP = addr.Address } - delete(availableCidr.IPv4Addresses, addr.Address) + //availableCidr.IPAddresses[addr.Address] = nil //Avoid mem leak - TODO + delete(availableCidr.IPAddresses, addr.Address) } } @@ -1186,19 +1365,19 @@ func (ds *DataStore) getUnusedIP(availableCidr *CidrInfo) (string, error) { ipnet := availableCidr.Cidr ip := availableCidr.Cidr.IP - for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); getNextIPv4Addr(ip) { + for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); getNextIPAddr(ip) { strPrivateIPv4 := ip.String() - if _, ok := availableCidr.IPv4Addresses[strPrivateIPv4]; ok { + if _, ok := availableCidr.IPAddresses[strPrivateIPv4]; ok { continue } ds.log.Debugf("Found a free IP not in DB - %s", strPrivateIPv4) return strPrivateIPv4, nil } - return "", errors.New("No free IP in the CIDR store") + return "", fmt.Errorf("no free IP available in the prefix - %s/%s", availableCidr.Cidr.IP, availableCidr.Cidr.Mask) } -func getNextIPv4Addr(ip net.IP) { +func getNextIPAddr(ip net.IP) { for j := len(ip) - 1; j >= 0; j-- { ip[j]++ if ip[j] > 0 { @@ -1230,11 +1409,12 @@ func (ds *DataStore) FindFreeableCidrs(eniID string) []CidrInfo { var freeable []CidrInfo for _, assignedaddr := range eni.AvailableIPv4Cidrs { - if assignedaddr.AssignedIPv4AddressesInCidr() == 0 { + if assignedaddr.AssignedIPAddressesInCidr() == 0 { tempFreeable := CidrInfo{ Cidr: assignedaddr.Cidr, - IPv4Addresses: nil, + IPAddresses: nil, IsPrefix: assignedaddr.IsPrefix, + AddressFamily: assignedaddr.AddressFamily, } freeable = append(freeable, tempFreeable) } diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 869938e0b1..07f959514b 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -255,7 +255,7 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, checkpoint.Data, &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IP: "1.1.1.1"}, }, }) @@ -285,7 +285,7 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, checkpoint.Data, &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IP: "1.1.1.1"}, }, }) checkpoint.Error = nil @@ -321,10 +321,10 @@ func TestPodIPv4Address(t *testing.T) { _, _, err = ds.AssignPodIPv4Address(key4) assert.Error(t, err) // Unassign unknown Pod - _, _, _, err = ds.UnassignPodIPv4Address(key4) + _, _, _, err = ds.UnassignPodIPAddress(key4) assert.Error(t, err) - _, _, deviceNum, err := ds.UnassignPodIPv4Address(key2) + _, _, deviceNum, err := ds.UnassignPodIPAddress(key2) assert.NoError(t, err) assert.Equal(t, ds.total, 3) assert.Equal(t, ds.assigned, 2) @@ -342,7 +342,7 @@ func TestPodIPv4Address(t *testing.T) { assert.True(t, eni == "") ds.eniPool["eni-2"].createTime = time.Time{} - ds.eniPool["eni-2"].AvailableIPv4Cidrs[ipv4Addr2.String()].IPv4Addresses["1.1.2.2"].UnassignedTime = time.Time{} + ds.eniPool["eni-2"].AvailableIPv4Cidrs[ipv4Addr2.String()].IPAddresses["1.1.2.2"].UnassignedTime = time.Time{} eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget, noWarmPrefixTarget) assert.Equal(t, eni, "eni-2") diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index a1ee87a523..a6c8b583d0 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -135,8 +135,18 @@ const ( //envWarmPrefixTarget is used to keep a /28 prefix in warm pool. envWarmPrefixTarget = "WARM_PREFIX_TARGET" defaultWarmPrefixTarget = 0 + + //envEnableIPv4 - Env variable to enable/disable IPv4 mode + envEnableIPv4 = "ENABLE_IPv4" + + //envEnableIPv6 - Env variable to enable/disable IPv6 mode + envEnableIPv6 = "ENABLE_IPv6" + + ipV4AddrFamily = "4" + ipV6AddrFamily = "6" ) + var log = logger.Get() var ( @@ -202,6 +212,8 @@ type IPAMContext struct { dataStore *datastore.DataStore rawK8SClient client.Client cachedK8SClient client.Client + enableIPv4 bool + enableIPv6 bool useCustomNetworking bool networkClient networkutils.NetworkAPIs maxIPsPerENI int @@ -217,12 +229,12 @@ type IPAMContext struct { lastDecreaseIPPool time.Time // reconcileCooldownCache keeps timestamps of the last time an IP address was unassigned from an ENI, // so that we don't reconcile and add it back too quickly if IMDS lags behind reality. - reconcileCooldownCache ReconcileCooldownCache - terminating int32 // Flag to warn that the pod is about to shut down. - disableENIProvisioning bool - enablePodENI bool - myNodeName string - enableIpv4PrefixDelegation bool + reconcileCooldownCache ReconcileCooldownCache + terminating int32 // Flag to warn that the pod is about to shut down. + disableENIProvisioning bool + enablePodENI bool + myNodeName string + enablePrefixDelegation bool } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -302,9 +314,11 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.cachedK8SClient = cachedK8SClient c.networkClient = networkutils.New() c.useCustomNetworking = UseCustomNetworkCfg() - c.enableIpv4PrefixDelegation = useIpv4PrefixDelegation() + c.enablePrefixDelegation = usePrefixDelegation() + c.enableIPv4 = isIPv4Enabled() + c.enableIPv6 = isIPv6Enabled() - client, err := awsutils.New(c.useCustomNetworking) + client, err := awsutils.New(c.useCustomNetworking, c.enableIPv4, c.enableIPv6) if err != nil { return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface") } @@ -312,6 +326,7 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.primaryIP = make(map[string]string) c.reconcileCooldownCache.cache = make(map[string]time.Time) + //WARM and Min IP/Prefix targets are ignored in IPv6 mode c.warmENITarget = getWarmENITarget() c.warmIPTarget = getWarmIPTarget() c.minimumIPTarget = getMinimumIPTarget() @@ -320,19 +335,16 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.disableENIProvisioning = disablingENIProvisioning() c.enablePodENI = enablePodENI() - hypervisorType, err := c.awsClient.GetInstanceHypervisorFamily() - if err != nil { - log.Error("Failed to get hypervisor type") + //Let's validate if the configured combination of env variables is supported before we + //proceed any further + if !c.isConfigValid() { return nil, err } - if hypervisorType != "nitro" && c.enableIpv4PrefixDelegation { - log.Warnf("Prefix delegation is not supported on non-nitro instance %s hence falling back to default (secondary IP) mode", c.awsClient.GetInstanceType()) - c.enableIpv4PrefixDelegation = false - } - c.awsClient.InitCachedPrefixDelegation(c.enableIpv4PrefixDelegation) + + c.awsClient.InitCachedPrefixDelegation(c.enablePrefixDelegation) c.myNodeName = os.Getenv("MY_NODE_NAME") checkpointer := datastore.NewJSONFile(dsBackingStorePath()) - c.dataStore = datastore.NewDataStore(log, checkpointer, c.enableIpv4PrefixDelegation) + c.dataStore = datastore.NewDataStore(log, checkpointer, c.enablePrefixDelegation) err = c.nodeInit() if err != nil { @@ -340,16 +352,18 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex } mac := c.awsClient.GetPrimaryENImac() - // retrieve security groups - err = c.awsClient.RefreshSGIDs(mac) - if err != nil { - return nil, err - } + if c.enableIPv4 { + // retrieve security groups + err = c.awsClient.RefreshSGIDs(mac) + if err != nil { + return nil, err + } - // Refresh security groups and VPC CIDR blocks in the background - // Ignoring errors since we will retry in 30s - go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second) + // Refresh security groups and VPC CIDR blocks in the background + // Ignoring errors since we will retry in 30s + go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second) + } return c, nil } @@ -357,28 +371,24 @@ func (c *IPAMContext) nodeInit() error { ipamdActionsInprogress.WithLabelValues("nodeInit").Add(float64(1)) defer ipamdActionsInprogress.WithLabelValues("nodeInit").Sub(float64(1)) var err error + var vpcV4CIDRs []string ctx := context.TODO() log.Debugf("Start node init") - nodeMaxENI, err := c.getMaxENI() - if err != nil { - log.Error("Failed to get ENI limit") - return err - } - c.maxENI = nodeMaxENI - c.maxIPsPerENI, c.maxPrefixesPerENI, err = c.GetIPv4Limit() - if err != nil { - return err + primaryV4IP := c.awsClient.GetLocalIPv4() + err = c.initENIAndIPLimits() + if c.enableIPv4 { + //Subnets currently will have both v4 and v6 CIDRs. Once EC2 launches v6 only Subnets, that will no longer + //be true and so it is safe (and only required) to get the v4 CIDR info only when IPv4 mode is enabled. + vpcV4CIDRs, err = c.awsClient.GetVPCIPv4CIDRs() + if err != nil { + return err + } } - log.Debugf("Max ip per ENI %d and max prefixes per ENI %d", c.maxIPsPerENI, c.maxPrefixesPerENI) - vpcCIDRs, err := c.awsClient.GetVPCIPv4CIDRs() - if err != nil { - return err - } - primaryIP := c.awsClient.GetLocalIPv4() - err = c.networkClient.SetupHostNetwork(vpcCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP, c.enablePodENI) + err = c.networkClient.SetupHostNetwork(vpcV4CIDRs, c.awsClient.GetPrimaryENImac(), &primaryV4IP, c.enablePodENI, c.enableIPv4, + c.enableIPv6) if err != nil { return errors.Wrap(err, "ipamd init: failed to set up host network") } @@ -394,10 +404,6 @@ func (c *IPAMContext) nodeInit() error { for _, eni := range enis { log.Debugf("Discovered ENI %s, trying to set it up", eni.ENIID) - if c.awsClient.IsCNIUnmanagedENI(eni.ENIID) { - log.Infof("Skipping ENI %s since it is not on network card 0", eni.ENIID) - continue - } isTrunkENI := eni.ENIID == metadataResult.TrunkENI isEFAENI := metadataResult.EFAENIs[eni.ENIID] @@ -433,11 +439,22 @@ func (c *IPAMContext) nodeInit() error { } } - if err := c.dataStore.ReadBackingStore(); err != nil { + if err := c.dataStore.ReadBackingStore(c.enableIPv6); err != nil { return err } - if c.enableIpv4PrefixDelegation { + + if c.enableIPv6 { + //We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always + //start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation + //feature in IPv6 mode as we don't support (yet) a non-PD v6 option. In addition, we don't support custom + //networking & SGPP in IPv6 mode yet. So, we will skip the corresponding setup. Will save us from checking + //if IPv6 is enabled at multiple places. Once we start supporting these features in IPv6 mode, we can do away + //with this check and not change anything else in the below setup. + return nil + } + + if c.enablePrefixDelegation { //During upgrade or if prefix delgation knob is disabled to enabled then we //might have secondary IPs attached to ENIs so doing a cleanup if not used before moving on c.tryUnassignIPsFromENIs() @@ -452,7 +469,7 @@ func (c *IPAMContext) nodeInit() error { } // Spawning updateCIDRsRulesOnChange go-routine go wait.Forever(func() { - vpcCIDRs = c.updateCIDRsRulesOnChange(vpcCIDRs) + vpcV4CIDRs = c.updateCIDRsRulesOnChange(vpcV4CIDRs) }, 30*time.Second) eniConfigName, err := eniconfig.GetNodeSpecificENIConfigName(ctx, c.cachedK8SClient) @@ -474,7 +491,6 @@ func (c *IPAMContext) nodeInit() error { } } - // If we started on a node with a trunk ENI already attached, add the node label. if metadataResult.TrunkENI != "" { // Signal to VPC Resource Controller that the node has a trunk already err := c.SetNodeLabel(ctx, "vpc.amazonaws.com/has-trunk-attached", "true") @@ -496,6 +512,7 @@ func (c *IPAMContext) nodeInit() error { } else if err != nil { return err } + return nil } @@ -531,7 +548,8 @@ func (c *IPAMContext) updateCIDRsRulesOnChange(oldVPCCIDRs []string) []string { new := sets.NewString(newVPCCIDRs...) if !old.Equal(new) { primaryIP := c.awsClient.GetLocalIPv4() - err = c.networkClient.UpdateHostIptablesRules(newVPCCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP) + err = c.networkClient.UpdateHostIptablesRules(newVPCCIDRs, c.awsClient.GetPrimaryENImac(), &primaryIP, c.enableIPv4, + c.enableIPv6) if err != nil { log.Warnf("unable to update host iptables rules for VPC CIDRs due to error: %v", err) } @@ -546,6 +564,11 @@ func (c *IPAMContext) updateIPStats(unmanaged int) { // StartNodeIPPoolManager monitors the IP pool, add or del them when it is required. func (c *IPAMContext) StartNodeIPPoolManager() { + if c.enableIPv6 { + //Nothing to do in IPv6 Mode. IPv6 is only supported in Prefix delegation mode + //and VPC CNI will only attach one V6 Prefix. + return + } sleepDuration := ipPoolMonitorInterval / 2 ctx := context.Background() for { @@ -587,9 +610,9 @@ func (c *IPAMContext) decreaseDatastorePool(interval time.Duration) { c.lastDecreaseIPPool = now c.lastNodeIPPoolAction = now - total, used, _ := c.dataStore.GetStats() + total, used, _ := c.dataStore.GetStats(ipV4AddrFamily) log.Debugf("Successfully decreased IP pool") - logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + logPoolStats(total, used, c.maxIPsPerENI, c.enablePrefixDelegation) } // tryFreeENI always tries to free one ENI @@ -714,13 +737,13 @@ func (c *IPAMContext) increaseDatastorePool(ctx context.Context) { func (c *IPAMContext) updateLastNodeIPPoolAction() { c.lastNodeIPPoolAction = time.Now() - total, used, totalPrefix := c.dataStore.GetStats() - if !c.enableIpv4PrefixDelegation { + total, used, totalPrefix := c.dataStore.GetStats(ipV4AddrFamily) + if !c.enablePrefixDelegation { log.Debugf("Successfully increased IP pool, total: %d, used: %d", total, used) - } else if c.enableIpv4PrefixDelegation { + } else if c.enablePrefixDelegation { log.Debugf("Successfully increased Prefix pool, total: %d, used: %d", totalPrefix, used) } - logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + logPoolStats(total, used, c.maxIPsPerENI, c.enablePrefixDelegation) } func (c *IPAMContext) tryAllocateENI(ctx context.Context) error { @@ -793,7 +816,7 @@ func (c *IPAMContext) tryAssignCidrs() (increasedPool bool, err error) { } } - if !c.enableIpv4PrefixDelegation { + if !c.enablePrefixDelegation { return c.tryAssignIPs() } else { return c.tryAssignPrefixes() @@ -843,6 +866,41 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { return false, nil } +func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) { + log.Debugf("Assigning an IPv6Prefix for ENI: %s", eniID) + //Let's make an EC2 API call to get a list of IPv6 prefixes (if any) that are already attached to the + //current ENI. We will make this call only once during boot up/init and doing so will shield us from any + //IMDS out of sync issues. We only need one v6 prefix per ENI/Node. + var ec2v6Prefixes []*ec2.Ipv6PrefixSpecification + ec2v6Prefixes, err = c.awsClient.GetIPv6PrefixesFromEC2(eniID) + if err != nil { + log.Errorf("assignIPv6Prefix; err: %s", err) + return err + } + log.Debugf("ENI %s has %v prefixe(s) attached", eniID, len(ec2v6Prefixes)) + + //Note: If we find more than one v6 prefix attached to the ENI, VPC CNI will not attempt to free it. VPC CNI + //will only attach a single v6 prefix and it will not attempt to free the additional Prefixes. + //We will add all the prefixes to our datastore. TODO - Should we instead pick one of them. If we do, how to track + //that across restarts? + + //Check if we already have v6 Prefix(es) attached + if len(ec2v6Prefixes) == 0 { + //Allocate and attach a v6 Prefix to Primary ENI + log.Debugf("No IPv6 Prefix(es) found for ENI: %s", eniID) + strPrefixes, err := c.awsClient.AllocIPv6Prefixes(eniID) + if err != nil { + return err + } + for _, v6Prefix := range strPrefixes { + ec2v6Prefixes = append(ec2v6Prefixes, &ec2.Ipv6PrefixSpecification{Ipv6Prefix: v6Prefix}) + } + log.Debugf("Successfully allocated an IPv6Prefix for ENI: %s", eniID) + } + c.addENIv6prefixesToDataStore(ec2v6Prefixes, eniID) + return nil +} + func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) { toAllocate := c.getPrefixesNeeded() // Returns an ENI which has space for more prefixes to be attached, but this @@ -865,7 +923,7 @@ func (c *IPAMContext) tryAssignPrefixes() (increasedPool bool, err error) { ipamdErrInc("increaseIPPoolGetENIprefixedFailed") return true, errors.Wrap(err, "failed to get ENI Prefix addresses during IPv4 Prefix allocation") } - c.addENIprefixesToDataStore(ec2Prefixes, eni.ID) + c.addENIv4prefixesToDataStore(ec2Prefixes, eni.ID) return true, nil } return false, nil @@ -885,25 +943,34 @@ func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, isT // Store the primary IP of the ENI c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address() - // For secondary ENIs, set up the network - if eni != primaryENI { - err = c.networkClient.SetupENINetwork(c.primaryIP[eni], eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.SubnetIPv4CIDR) + if c.enableIPv6 && eni == primaryENI { + //In v6 PD Mode, VPC CNI will only manage primary ENI. Once we start supporting secondary IP and custom + //networking modes for v6, we will relax this restriction. We filter out all the ENIs except Primary ENI + //in v6 mode (prior to landing here), but included the primary ENI check as a safety net. + err := c.assignIPv6Prefix(eni) if err != nil { - // Failed to set up the ENI - errRemove := c.dataStore.RemoveENIFromDataStore(eni, true) - if errRemove != nil { - log.Warnf("failed to remove ENI %s: %v", eni, errRemove) + return errors.Wrapf(err, "Failed to allocate IPv6 Prefixes to Primary ENI") + } + } else { + // For secondary ENIs, set up the network + if eni != primaryENI { + err = c.networkClient.SetupENINetwork(c.primaryIP[eni], eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.SubnetIPv4CIDR) + if err != nil { + // Failed to set up the ENI + errRemove := c.dataStore.RemoveENIFromDataStore(eni, true) + if errRemove != nil { + log.Warnf("failed to remove ENI %s: %v", eni, errRemove) + } + delete(c.primaryIP, eni) + return errors.Wrapf(err, "failed to set up ENI %s network", eni) } - delete(c.primaryIP, eni) - return errors.Wrapf(err, "failed to set up ENI %s network", eni) } + log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes)) + //Either case add the IPs and prefixes to datastore. + c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni) + c.addENIv4prefixesToDataStore(eniMetadata.IPv4Prefixes, eni) } - log.Infof("Found ENIs having %d secondary IPs and %d Prefixes", len(eniMetadata.IPv4Addresses), len(eniMetadata.IPv4Prefixes)) - //Either case add the IPs and prefixes to datastore. - c.addENIsecondaryIPsToDataStore(eniMetadata.IPv4Addresses, eni) - c.addENIprefixesToDataStore(eniMetadata.IPv4Prefixes, eni) - return nil } @@ -922,11 +989,11 @@ func (c *IPAMContext) addENIsecondaryIPsToDataStore(ec2PrivateIpAddrs []*ec2.Net } } - total, assigned, totalPrefix := c.dataStore.GetStats() + total, assigned, totalPrefix := c.dataStore.GetStats(ipV4AddrFamily) log.Debugf("Datastore Pool stats: total(/32): %d, assigned(/32): %d, total prefixes(/28): %d", total, assigned, totalPrefix) } -func (c *IPAMContext) addENIprefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4PrefixSpecification, eni string) { +func (c *IPAMContext) addENIv4prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4PrefixSpecification, eni string) { //Walk thru all prefixes for _, ec2PrefixAddr := range ec2PrefixAddrs { @@ -942,13 +1009,36 @@ func (c *IPAMContext) addENIprefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv4Prefix if err != nil && err.Error() != datastore.IPAlreadyInStoreError { log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store", ec2PrefixAddr.Ipv4Prefix) // continue to add next address - ipamdErrInc("addENIprefixesToDataStoreFailed") + ipamdErrInc("addENIv4prefixesToDataStoreFailed") } } - total, assigned, totalPrefix := c.dataStore.GetStats() + total, assigned, totalPrefix := c.dataStore.GetStats(ipV4AddrFamily) log.Debugf("Datastore Pool stats: total(/32): %d, assigned(/32): %d, total prefixes(/28): %d", total, assigned, totalPrefix) } +func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv6PrefixSpecification, eni string) { + log.Debugf("Updating datastore with IPv6Prefix(es) for ENI: %v, count: %v", eni, len(ec2PrefixAddrs)) + //Walk through all prefixes + for _, ec2PrefixAddr := range ec2PrefixAddrs { + strIpv6Prefix := aws.StringValue(ec2PrefixAddr.Ipv6Prefix) + _, ipnet, err := net.ParseCIDR(strIpv6Prefix) + if err != nil { + //Parsing failed, get next prefix + log.Debugf("Parsing failed, moving on to next prefix") + continue + } + cidr := *ipnet + err = c.dataStore.AddIPv6CidrToStore(eni, cidr, true) + if err != nil && err.Error() != datastore.IPAlreadyInStoreError { + log.Warnf("Failed to increase Prefix pool, failed to add Prefix %s to data store", ec2PrefixAddr.Ipv6Prefix) + // continue to add next address + ipamdErrInc("addENIv6prefixesToDataStoreFailed") + } + } + _, _, totalPrefix := c.dataStore.GetStats(ipV6AddrFamily) + log.Debugf("Datastore Pool stats: total v6 prefixes(/80): %d", totalPrefix) +} + // getMaxENI returns the maximum number of ENIs to attach to this instance. This is calculated as the lesser of // the limit for the instance type and the value configured via the MAX_ENI environment variable. If the value of // the environment variable is 0 or less, it will be ignored and the maximum for the instance is returned. @@ -1041,23 +1131,23 @@ func (c *IPAMContext) shouldRemoveExtraENIs() bool { return true } - total, used, _ := c.dataStore.GetStats() + total, used, _ := c.dataStore.GetStats(ipV4AddrFamily) available := total - used var shouldRemoveExtra bool // We need the +1 to make sure we are not going below the WARM_ENI_TARGET/WARM_PREFIX_TARGET warmTarget := (c.warmENITarget + 1) - if c.enableIpv4PrefixDelegation { + if c.enablePrefixDelegation { warmTarget = (c.warmPrefixTarget + 1) } shouldRemoveExtra = available >= (warmTarget)*c.maxIPsPerENI if shouldRemoveExtra { - logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + logPoolStats(total, used, c.maxIPsPerENI, c.enablePrefixDelegation) log.Debugf("It might be possible to remove extra ENIs because available (%d) >= (ENI/Prefix target + 1 (%d) + 1) * addrsPerENI (%d)", available, warmTarget, c.maxIPsPerENI) - } else if c.enableIpv4PrefixDelegation { + } else if c.enablePrefixDelegation { // When prefix target count is reduced, datastorehigh would have deleted extra prefixes over the warm prefix target. // Hence available will be less than (warmTarget)*c.maxIPsPerENI but there can be some extra ENIs which are not used hence see if we can clean it up. shouldRemoveExtra = c.dataStore.CheckFreeableENIexists() @@ -1071,13 +1161,13 @@ func (c *IPAMContext) computeExtraPrefixesOverWarmTarget() int { return over } - total, used, _ := c.dataStore.GetStats() + total, used, _ := c.dataStore.GetStats(ipV4AddrFamily) available := total - used freePrefixes := c.dataStore.GetFreePrefixes() over = max(freePrefixes-c.warmPrefixTarget, 0) - logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + logPoolStats(total, used, c.maxIPsPerENI, c.enablePrefixDelegation) log.Debugf("computeExtraPrefixesOverWarmTarget available %d over %d warm_prefix_target %d", available, over, c.warmPrefixTarget) return over @@ -1212,7 +1302,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur reconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc() } log.Debug("Successfully Reconciled ENI/IP pool") - total, assigned, totalPrefix := c.dataStore.GetStats() + total, assigned, totalPrefix := c.dataStore.GetStats(ipV4AddrFamily) log.Debugf("IP/Prefix Address Pool stats: total: %d, assigned: %d, total prefixes: %d", total, assigned, totalPrefix) c.lastNodeIPPoolAction = curTime } @@ -1506,17 +1596,32 @@ func enablePodENI() bool { return getEnvBoolWithDefault(envEnablePodENI, false) } -func useIpv4PrefixDelegation() bool { +func usePrefixDelegation() bool { return getEnvBoolWithDefault(envEnableIpv4PrefixDelegation, false) } +func isIPv4Enabled() bool { + return getEnvBoolWithDefault(envEnableIPv4, false) +} + +func isIPv6Enabled() bool { + return getEnvBoolWithDefault(envEnableIPv6, false) +} + // filterUnmanagedENIs filters out ENIs marked with the "node.k8s.amazonaws.com/no_manage" tag func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutils.ENIMetadata { numFiltered := 0 ret := make([]awsutils.ENIMetadata, 0, len(enis)) for _, eni := range enis { - // If we have unmanaged ENIs, filter them out - if c.awsClient.IsUnmanagedENI(eni.ENIID) { + //Filter out any Unmanaged ENIs. VPC CNI will only work with Primary ENI in IPv6 Prefix Delegation mode until + //we open up IPv6 support in Secondary IP and Custom networking modes. Filtering out the ENIs here will + //help us avoid myriad of if/else loops elsewhere in the code. + if c.enableIPv6 && !c.awsClient.IsPrimaryENI(eni.ENIID) { + log.Debugf("Skipping ENI %s: IPv6 Mode is enabled and VPC CNI will only manage Primary ENI in v6 PD mode", + eni.ENIID) + numFiltered++ + continue + } else if c.awsClient.IsUnmanagedENI(eni.ENIID) { log.Debugf("Skipping ENI %s: tagged with %s", eni.ENIID, eniNoManageTagKey) numFiltered++ continue @@ -1525,7 +1630,6 @@ func (c *IPAMContext) filterUnmanagedENIs(enis []awsutils.ENIMetadata) []awsutil numFiltered++ continue } - ret = append(ret, eni) } c.unmanagedENI = numFiltered @@ -1543,7 +1647,7 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) return 0, 0, false } - total, assigned, totalPrefix := c.dataStore.GetStats() + total, assigned, totalPrefix := c.dataStore.GetStats(ipV4AddrFamily) available := total - assigned // short is greater than 0 when we have fewer available IPs than the warm IP target @@ -1558,7 +1662,7 @@ func (c *IPAMContext) datastoreTargetState() (short int, over int, enabled bool) // over is less than the warm IP target alone if it would imply reducing total IPs below the minimum target over = max(min(over, total-c.minimumIPTarget), 0) - if c.enableIpv4PrefixDelegation { + if c.enablePrefixDelegation { //short : number of IPs short to reach warm targets //over : number of IPs over the warm targets @@ -1784,7 +1888,7 @@ func (c *IPAMContext) tryUnassignPrefixFromENI(eniID string) { func (c *IPAMContext) GetENIResourcesToAllocate() int { var resourcesToAllocate int - if !c.enableIpv4PrefixDelegation { + if !c.enablePrefixDelegation { resourcesToAllocate = c.maxIPsPerENI short, _, warmTargetDefined := c.datastoreTargetState() if warmTargetDefined { @@ -1799,13 +1903,13 @@ func (c *IPAMContext) GetENIResourcesToAllocate() int { func (c *IPAMContext) GetIPv4Limit() (int, int, error) { var maxIPsPerENI, maxPrefixesPerENI, maxIpsPerPrefix int var err error - if !c.enableIpv4PrefixDelegation { + if !c.enablePrefixDelegation { maxIPsPerENI, err = c.awsClient.GetENIIPv4Limit() maxPrefixesPerENI = 0 if err != nil { return 0, 0, err } - } else if c.enableIpv4PrefixDelegation { + } else if c.enablePrefixDelegation { //Single PD - allocate one prefix per ENI and new add will be new ENI + prefix //Multi - allocate one prefix per ENI and new add will be new prefix or new ENI + prefix _, maxIpsPerPrefix, _ = datastore.GetPrefixDelegationDefaults() @@ -1825,13 +1929,13 @@ func (c *IPAMContext) isDatastorePoolTooLow() bool { return short > 0 } - total, used, _ := c.dataStore.GetStats() + total, used, _ := c.dataStore.GetStats(ipV4AddrFamily) available := total - used warmTarget := c.warmENITarget totalIPs := c.maxIPsPerENI - if c.enableIpv4PrefixDelegation { + if c.enablePrefixDelegation { warmTarget = c.warmPrefixTarget _, maxIpsPerPrefix, _ := datastore.GetPrefixDelegationDefaults() totalIPs = maxIpsPerPrefix @@ -1839,7 +1943,7 @@ func (c *IPAMContext) isDatastorePoolTooLow() bool { poolTooLow := available < totalIPs*warmTarget || (warmTarget == 0 && available == 0) if poolTooLow { - logPoolStats(total, used, c.maxIPsPerENI, c.enableIpv4PrefixDelegation) + logPoolStats(total, used, c.maxIPsPerENI, c.enablePrefixDelegation) log.Debugf("IP pool is too low: available (%d) < ENI target (%d) * addrsPerENI (%d)", available, warmTarget, totalIPs) } return poolTooLow @@ -1866,7 +1970,7 @@ func (c *IPAMContext) isDatastorePoolTooHigh() bool { } func (c *IPAMContext) warmPrefixTargetDefined() bool { - return c.warmPrefixTarget >= defaultWarmPrefixTarget && c.enableIpv4PrefixDelegation + return c.warmPrefixTarget >= defaultWarmPrefixTarget && c.enablePrefixDelegation } //DeallocCidrs frees IPs and Prefixes from EC2 @@ -1920,3 +2024,64 @@ func (c *IPAMContext) getPrefixesNeeded() int { log.Debugf("ToAllocate: %d", toAllocate) return toAllocate } + +func (c *IPAMContext) initENIAndIPLimits() (err error) { + if c.enableIPv4 { + nodeMaxENI, err := c.getMaxENI() + if err != nil { + log.Error("Failed to get ENI limit") + return err + } + c.maxENI = nodeMaxENI + + c.maxIPsPerENI, c.maxPrefixesPerENI, err = c.GetIPv4Limit() + if err != nil { + return err + } + log.Debugf("Max ip per ENI %d and max prefixes per ENI %d", c.maxIPsPerENI, c.maxPrefixesPerENI) + } + + //WARM and MAX ENI & IP/Prefix counts are no-op in IPv6 Prefix delegation mode. Once we start supporting IPv6 in + //Secondary IP mode, these variables will play the same role as they currently do in IPv4 mode. So, for now we + //leave them at their default values. + + return nil +} + +func (c *IPAMContext) isConfigValid() bool { + //Get Instance type + hypervisorType, err := c.awsClient.GetInstanceHypervisorFamily() + if err != nil { + log.Error("Failed to get hypervisor type") + return false + } + + //Validate that only one among v4 and v6 is enabled. + if c.enableIPv4 && c.enableIPv6 { + log.Errorf("IPv4 and IPv6 are both enabled. VPC CNI currently doesn't support dual stack mode") + return false + } else if !c.enableIPv4 && !c.enableIPv6 { + log.Errorf("IPv4 and IPv6 are both disabled. One of them have to be enabled") + return false + } + + //Validate PD mode is enabled if VPC CNI is operating in IPv6 mode. SGPP and Custom networking are not supported in IPv6 mode. + if c.enableIPv6 && (c.enablePodENI || c.useCustomNetworking || !c.enablePrefixDelegation) { + log.Errorf("IPv6 is supported only in Prefix Delegation mode. Security Group Per Pod and " + + "Custom Networking are not supported in IPv6 mode. Please set the env variables accordingly.") + return false + } + + //Validate Prefix Delegation against v4 and v6 modes. + if hypervisorType != "nitro" && c.enablePrefixDelegation { + if c.enableIPv6 { + log.Errorf("Prefix Delegation is not supported on non-nitro instance %s. IPv6 is only supported in Prefix delegation Mode. ", c.awsClient.GetInstanceType()) + return false + } + log.Warnf("Prefix delegation is not supported on non-nitro instance %s hence falling back to default (secondary IP) mode", c.awsClient.GetInstanceType()) + c.enablePrefixDelegation = false + } + + + return true +} diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index 4ca8085316..d109a5a4cf 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -65,6 +65,8 @@ const ( prefix02 = "10.10.40.0/28" ipaddrPD01 = "10.10.30.0" ipaddrPD02 = "10.10.40.0" + v6ipaddr01 = "2001:db8::1/128" + v6prefix01 = "2001:db8::/64" ) type testMocks struct { @@ -100,7 +102,7 @@ func TestNodeInit(t *testing.T) { fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddr02}, + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddr02}, }, } @@ -117,6 +119,8 @@ func TestNodeInit(t *testing.T) { networkClient: m.network, dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), false), myNodeName: myNodeName, + enableIPv4: true, + enableIPv6: false, } mockContext.dataStore.CheckpointMigrationPhase = 2 @@ -136,7 +140,7 @@ func TestNodeInit(t *testing.T) { primaryIP := net.ParseIP(ipaddr01) m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") - m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false).Return(nil) + m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) @@ -174,7 +178,7 @@ func TestNodeInit(t *testing.T) { assert.NoError(t, err) } -func TestNodeInitwithPDenabled(t *testing.T) { +func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { m := setup(t) defer m.ctrl.Finish() ctx := context.Background() @@ -182,25 +186,27 @@ func TestNodeInitwithPDenabled(t *testing.T) { fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddrPD01}, + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddrPD01}, }, } mockContext := &IPAMContext{ - awsClient: m.awsutils, - rawK8SClient: m.rawK8SClient, - cachedK8SClient: m.cachedK8SClient, - maxIPsPerENI: 224, - maxPrefixesPerENI: 14, - maxENI: 4, - warmENITarget: 1, - warmIPTarget: 3, - primaryIP: make(map[string]string), - terminating: int32(0), - networkClient: m.network, - dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true), - myNodeName: myNodeName, - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, + maxIPsPerENI: 224, + maxPrefixesPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true), + myNodeName: myNodeName, + enablePrefixDelegation: true, + enableIPv4: true, + enableIPv6: false, } mockContext.dataStore.CheckpointMigrationPhase = 2 @@ -220,7 +226,7 @@ func TestNodeInitwithPDenabled(t *testing.T) { primaryIP := net.ParseIP(ipaddr01) m.awsutils.EXPECT().GetVPCIPv4CIDRs().AnyTimes().Return(cidrs, nil) m.awsutils.EXPECT().GetPrimaryENImac().Return("") - m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false).Return(nil) + m.network.EXPECT().SetupHostNetwork(cidrs, "", &primaryIP, false, true, false).Return(nil) m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) @@ -255,6 +261,76 @@ func TestNodeInitwithPDenabled(t *testing.T) { assert.NoError(t, err) } +func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + ctx := context.Background() + + fakeCheckpoint := datastore.CheckpointData{ + Version: datastore.CheckpointFormatVersion, + Allocations: []datastore.CheckpointEntry{ + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddrPD01}, + }, + } + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, + maxIPsPerENI: 224, + maxPrefixesPerENI: 1, + maxENI: 1, + warmENITarget: 1, + warmIPTarget: 1, + primaryIP: make(map[string]string), + terminating: int32(0), + networkClient: m.network, + dataStore: datastore.NewDataStore(log, datastore.NewTestCheckpoint(fakeCheckpoint), true), + myNodeName: myNodeName, + enablePrefixDelegation: true, + enableIPv4: false, + enableIPv6: true, + } + mockContext.dataStore.CheckpointMigrationPhase = 2 + + eni1 := getDummyENIMetadataWithV6Prefix() + + var cidrs []string + m.awsutils.EXPECT().IsUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + m.awsutils.EXPECT().TagENI(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + m.awsutils.EXPECT().IsCNIUnmanagedENI(eni1.ENIID).Return(false).AnyTimes() + + primaryIP := net.ParseIP(ipaddr01) + m.network.EXPECT().SetupHostNetwork(cidrs, eni1.MAC, &primaryIP, false, false, true).Return(nil) + m.awsutils.EXPECT().GetIPv6PrefixesFromEC2(eni1.ENIID).AnyTimes().Return(eni1.IPv6Prefixes, nil) + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().GetPrimaryENImac().Return(eni1.MAC) + m.awsutils.EXPECT().IsPrimaryENI(primaryENIid).Return(true).AnyTimes() + + + eniMetadataSlice := []awsutils.ENIMetadata{eni1} + resp := awsutils.DescribeAllENIsResult{ + ENIMetadata: eniMetadataSlice, + TagMap: map[string]awsutils.TagMap{}, + TrunkENI: "", + EFAENIs: make(map[string]bool), + } + m.awsutils.EXPECT().DescribeAllENIs().Return(resp, nil) + m.awsutils.EXPECT().GetLocalIPv4().Return(primaryIP) + m.awsutils.EXPECT().SetCNIUnmanagedENIs(resp.MultiCardENIIDs).AnyTimes() + + fakeNode := v1.Node{ + TypeMeta: metav1.TypeMeta{Kind: "Node"}, + ObjectMeta: metav1.ObjectMeta{Name: myNodeName}, + Spec: v1.NodeSpec{}, + Status: v1.NodeStatus{}, + } + _ = m.cachedK8SClient.Create(ctx, &fakeNode) + + err := mockContext.nodeInit() + assert.NoError(t, err) +} + func getDummyENIMetadata() (awsutils.ENIMetadata, awsutils.ENIMetadata) { primary := true notPrimary := false @@ -330,6 +406,30 @@ func getDummyENIMetadataWithPrefix() (awsutils.ENIMetadata, awsutils.ENIMetadata return eni1, eni2 } +func getDummyENIMetadataWithV6Prefix() awsutils.ENIMetadata { + primary := true + testAddr1 := v6ipaddr01 + testv6Prefix := v6prefix01 + eni1 := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + IPv6Prefixes: []*ec2.Ipv6PrefixSpecification{ + { + Ipv6Prefix: &testv6Prefix, + }, + }, + } + + return eni1 +} + func TestIncreaseIPPoolDefault(t *testing.T) { _ = os.Unsetenv(envCustomNetworkCfg) testIncreaseIPPool(t, false) @@ -468,19 +568,19 @@ func testIncreasePrefixPool(t *testing.T, useENIConfig bool) { ctx := context.Background() mockContext := &IPAMContext{ - awsClient: m.awsutils, - rawK8SClient: m.rawK8SClient, - cachedK8SClient: m.cachedK8SClient, - maxIPsPerENI: 256, - maxPrefixesPerENI: 16, - maxENI: 4, - warmENITarget: 1, - warmPrefixTarget: 1, - networkClient: m.network, - useCustomNetworking: UseCustomNetworkCfg(), - primaryIP: make(map[string]string), - terminating: int32(0), - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + rawK8SClient: m.rawK8SClient, + cachedK8SClient: m.cachedK8SClient, + maxIPsPerENI: 256, + maxPrefixesPerENI: 16, + maxENI: 4, + warmENITarget: 1, + warmPrefixTarget: 1, + networkClient: m.network, + useCustomNetworking: UseCustomNetworkCfg(), + primaryIP: make(map[string]string), + terminating: int32(0), + enablePrefixDelegation: true, } mockContext.dataStore = testDatastorewithPrefix() @@ -749,11 +849,11 @@ func TestNodePrefixPoolReconcile(t *testing.T) { ctx := context.Background() mockContext := &IPAMContext{ - awsClient: m.awsutils, - networkClient: m.network, - primaryIP: make(map[string]string), - terminating: int32(0), - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enablePrefixDelegation: true, } mockContext.dataStore = testDatastorewithPrefix() @@ -931,11 +1031,11 @@ func TestGetWarmIPTargetStatewithPDenabled(t *testing.T) { defer m.ctrl.Finish() mockContext := &IPAMContext{ - awsClient: m.awsutils, - networkClient: m.network, - primaryIP: make(map[string]string), - terminating: int32(0), - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enablePrefixDelegation: true, } mockContext.dataStore = testDatastorewithPrefix() @@ -1000,15 +1100,15 @@ func TestIPAMContext_nodeIPPoolTooLow(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &IPAMContext{ - awsClient: m.awsutils, - dataStore: tt.fields.datastore, - useCustomNetworking: false, - networkClient: m.network, - maxIPsPerENI: tt.fields.maxIPsPerENI, - maxENI: -1, - warmENITarget: tt.fields.warmENITarget, - warmIPTarget: tt.fields.warmIPTarget, - enableIpv4PrefixDelegation: false, + awsClient: m.awsutils, + dataStore: tt.fields.datastore, + useCustomNetworking: false, + networkClient: m.network, + maxIPsPerENI: tt.fields.maxIPsPerENI, + maxENI: -1, + warmENITarget: tt.fields.warmENITarget, + warmIPTarget: tt.fields.warmIPTarget, + enablePrefixDelegation: false, } if got := c.isDatastorePoolTooLow(); got != tt.want { t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) @@ -1044,15 +1144,15 @@ func TestIPAMContext_nodePrefixPoolTooLow(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &IPAMContext{ - awsClient: m.awsutils, - dataStore: tt.fields.datastore, - useCustomNetworking: false, - networkClient: m.network, - maxPrefixesPerENI: tt.fields.maxPrefixesPerENI, - maxIPsPerENI: tt.fields.maxIPsPerENI, - maxENI: -1, - warmPrefixTarget: tt.fields.warmPrefixTarget, - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + dataStore: tt.fields.datastore, + useCustomNetworking: false, + networkClient: m.network, + maxPrefixesPerENI: tt.fields.maxPrefixesPerENI, + maxIPsPerENI: tt.fields.maxIPsPerENI, + maxENI: -1, + warmPrefixTarget: tt.fields.warmPrefixTarget, + enablePrefixDelegation: true, } if got := c.isDatastorePoolTooLow(); got != tt.want { t.Errorf("nodeIPPoolTooLow() = %v, want %v", got, tt.want) @@ -1313,11 +1413,11 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) { ctx := context.Background() mockContext := &IPAMContext{ - awsClient: m.awsutils, - networkClient: m.network, - primaryIP: make(map[string]string), - terminating: int32(0), - enableIpv4PrefixDelegation: true, + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + enablePrefixDelegation: true, } mockContext.dataStore = testDatastorewithPrefix() @@ -1328,7 +1428,7 @@ func TestNodePrefixPoolReconcileBadIMDSData(t *testing.T) { eniID := primaryENIMetadata.ENIID _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false, false) mockContext.primaryIP[eniID] = testAddr1 - mockContext.addENIprefixesToDataStore(primaryENIMetadata.IPv4Prefixes, eniID) + mockContext.addENIv4prefixesToDataStore(primaryENIMetadata.IPv4Prefixes, eniID) curENIs := mockContext.dataStore.GetENIInfos() assert.Equal(t, 1, len(curENIs.ENIs)) assert.Equal(t, 16, curENIs.TotalIPs) @@ -1637,3 +1737,135 @@ func TestIPAMContext_askForTrunkENIIfNeeded(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "false", updatedNode.Labels["vpc.amazonaws.com/has-trunk-attached"]) } + +func TestIsConfigValid(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + type fields struct { + ipV4Enabled bool + ipV6Enabled bool + prefixDelegationEnabled bool + customNetworkingEnabled bool + podENIEnabled bool + isNitroInstance bool + } + + tests := []struct { + name string + fields fields + want bool + }{ + { + name: "v4 enabled in non-PD mode and v6 disabled", + fields: fields{ + ipV4Enabled: true, + ipV6Enabled: false, + prefixDelegationEnabled: false, + isNitroInstance: true, + }, + want: true, + }, + { + name: "v4 enabled in PD mode and v6 disabled", + fields: fields{ + ipV4Enabled: true, + ipV6Enabled: false, + prefixDelegationEnabled: true, + isNitroInstance: true, + }, + want: true, + }, + { + name: "v4 disabled and v6 enabled in PD mode", + fields: fields{ + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: true, + isNitroInstance: true, + }, + want: true, + }, + { + name: "v4 disabled and v6 enabled in non-PD mode", + fields: fields{ + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: false, + isNitroInstance: true, + }, + want: false, + }, + { + name: "both v4 and v6 enabled", + fields: fields{ + ipV4Enabled: true, + ipV6Enabled: true, + isNitroInstance: true, + }, + want: false, + }, + { + name: "v4 disabled and v6 enabled in PD mode on Non-Nitro instance", + fields: fields{ + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: true, + isNitroInstance: false, + }, + want: false, + }, + { + name: "ppsg enabled in v6 mode", + fields: fields{ + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: true, + podENIEnabled: true, + isNitroInstance: true, + }, + want: false, + }, + { + name: "ppsg enabled in v4 mode", + fields: fields{ + ipV4Enabled: true, + ipV6Enabled: false, + prefixDelegationEnabled: true, + podENIEnabled: true, + isNitroInstance: true, + }, + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + if tt.fields.isNitroInstance { + m.awsutils.EXPECT().GetInstanceHypervisorFamily().Return("nitro", nil) + } else { + m.awsutils.EXPECT().GetInstanceType().Return("dummy-instance") + m.awsutils.EXPECT().GetInstanceHypervisorFamily().Return("non-nitro", nil) + } + ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled) + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + enableIPv4: tt.fields.ipV4Enabled, + enableIPv6: tt.fields.ipV6Enabled, + enablePrefixDelegation: tt.fields.prefixDelegationEnabled, + enablePodENI: tt.fields.podENIEnabled, + useCustomNetworking: tt.fields.customNetworkingEnabled, + dataStore: ds, + } + + resp := mockContext.isConfigValid() + assert.Equal(t, tt.want, resp) + }) + } + +} diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index a4bc529197..0402904d7f 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -72,9 +72,9 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp failureResponse := rpc.AddNetworkReply{Success: false} var deviceNumber, vlanID, trunkENILinkIndex int - var addr, branchENIMAC, podENISubnetGW string + var ipv4Addr, ipv6Addr, branchENIMAC, podENISubnetGW string var err error - if s.ipamContext.enablePodENI { + if !s.ipamContext.enableIPv6 && s.ipamContext.enablePodENI { // Check pod spec for Branch ENI pod, err := s.ipamContext.GetPod(in.K8S_POD_NAME, in.K8S_POD_NAMESPACE) if err != nil { @@ -105,10 +105,10 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp return &failureResponse, nil } firstENI := podENIData[0] - addr = firstENI.PrivateIP + ipv4Addr = firstENI.PrivateIP branchENIMAC = firstENI.IfAddress vlanID = firstENI.VlanID - if addr == "" || branchENIMAC == "" || vlanID == 0 { + if ipv4Addr == "" || branchENIMAC == "" || vlanID == 0 { log.Errorf("Failed to parse pod-ENI annotation: %s", val) return &failureResponse, nil } @@ -128,7 +128,9 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp } } } - if addr == "" { + + if s.ipamContext.enableIPv4 && ipv4Addr == "" || + s.ipamContext.enableIPv6 && ipv6Addr == "" { if in.ContainerID == "" || in.IfName == "" || in.NetworkName == "" { log.Errorf("Unable to generate IPAMKey from %+v", in) return &failureResponse, nil @@ -138,43 +140,52 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp IfName: in.IfName, NetworkName: in.NetworkName, } - addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPv4Address(ipamKey) - if err != nil { - log.Warnf("Send AddNetworkReply: unable to assign IPv4 address for pod, err: %v", err) - return &failureResponse, nil - } - } - - pbVPCcidrs, err := s.ipamContext.awsClient.GetVPCIPv4CIDRs() - if err != nil { - log.Errorf("Send AddNetworkReply: unable to obtain VPC CIDRs, err: %v", err) - return &failureResponse, nil - } - for _, cidr := range pbVPCcidrs { - log.Debugf("VPC CIDR %s", cidr) + //addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPv4Address(ipamKey) + ipv4Addr, ipv6Addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPAddress(ipamKey, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6) } - useExternalSNAT := s.ipamContext.networkClient.UseExternalSNAT() - if !useExternalSNAT { - for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() { - log.Debugf("CIDR SNAT Exclusion %s", cidr) - pbVPCcidrs = append(pbVPCcidrs, cidr) + var pbVPCV4cidrs, pbVPCV6cidrs []string + var useExternalSNAT bool + if s.ipamContext.enableIPv4 && ipv4Addr != "" { + pbVPCV4cidrs, err = s.ipamContext.awsClient.GetVPCIPv4CIDRs() + if err != nil { + return nil, err + } + for _, cidr := range pbVPCV4cidrs { + log.Debugf("VPC CIDR %s", cidr) + } + useExternalSNAT = s.ipamContext.networkClient.UseExternalSNAT() + if !useExternalSNAT { + for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() { + log.Debugf("CIDR SNAT Exclusion %s", cidr) + pbVPCV4cidrs = append(pbVPCV4cidrs, cidr) + } + } + } else if s.ipamContext.enableIPv6 && ipv6Addr != "" { + pbVPCV6cidrs, err = s.ipamContext.awsClient.GetVPCIPv6CIDRs() + if err != nil { + return nil, err + } + for _, cidr := range pbVPCV6cidrs { + log.Debugf("VPC V6 CIDR %s", cidr) } } resp := rpc.AddNetworkReply{ Success: err == nil, - IPv4Addr: addr, + IPv4Addr: ipv4Addr, + IPv6Addr: ipv6Addr, DeviceNumber: int32(deviceNumber), UseExternalSNAT: useExternalSNAT, - VPCcidrs: pbVPCcidrs, + VPCV4Cidrs: pbVPCV4cidrs, + VPCV6Cidrs: pbVPCV6cidrs, PodVlanId: int32(vlanID), PodENIMAC: branchENIMAC, PodENISubnetGW: podENISubnetGW, ParentIfIndex: int32(trunkENILinkIndex), } - log.Infof("Send AddNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", addr, deviceNumber, err) + log.Infof("Send AddNetworkReply: IPv4Addr %s, IPv6Addr: %s, DeviceNumber: %d, err: %v", ipv4Addr, ipv6Addr, deviceNumber, err) return &resp, nil } @@ -201,19 +212,19 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp IfName: in.IfName, NetworkName: in.NetworkName, } - eni, ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPv4Address(ipamKey) + eni, ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPAddress(ipamKey) cidr := net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)} cidrStr := cidr.String() - if eni != nil { - //cidrStr will be pod IP i.e, IP/32. + if s.ipamContext.enableIPv4 && eni != nil { + //cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6. // Case 1: PD is enabled but IP/32 key in AvailableIPv4Cidrs[cidrStr] exists, this means it is a secondary IP. Added IsPrefix check just for sanity. // So this IP should be released immediately. // Case 2: PD is disabled then IP/32 key in AvailableIPv4Cidrs[cidrStr] will not exists since key to AvailableIPv4Cidrs will be either /28 prefix or /32 // secondary IP. Hence now see if we need free up a prefix is no other pods are using it. - if s.ipamContext.enableIpv4PrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] != nil && eni.AvailableIPv4Cidrs[cidrStr].IsPrefix == false { + if s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] != nil && eni.AvailableIPv4Cidrs[cidrStr].IsPrefix == false { log.Debugf("IP belongs to secondary pool with PD enabled so free IP from EC2") s.ipamContext.tryUnassignIPFromENI(eni.ID) - } else if !s.ipamContext.enableIpv4PrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] == nil { + } else if !s.ipamContext.enablePrefixDelegation && eni.AvailableIPv4Cidrs[cidrStr] == nil { log.Debugf("IP belongs to prefix pool with PD disabled so try free prefix from EC2") s.ipamContext.tryUnassignPrefixFromENI(eni.ID) } diff --git a/pkg/ipamd/rpc_handler_test.go b/pkg/ipamd/rpc_handler_test.go index 5299343b84..eae015ce3b 100644 --- a/pkg/ipamd/rpc_handler_test.go +++ b/pkg/ipamd/rpc_handler_test.go @@ -39,6 +39,7 @@ func TestServer_VersionCheck(t *testing.T) { dataStore: datastore.NewDataStore(log, datastore.NullCheckpoint{}, false), } m.awsutils.EXPECT().GetVPCIPv4CIDRs().Return([]string{}, nil).AnyTimes() + m.awsutils.EXPECT().GetVPCIPv6CIDRs().Return([]string{}, nil).AnyTimes() m.network.EXPECT().UseExternalSNAT().Return(true).AnyTimes() rpcServer := server{ @@ -84,6 +85,10 @@ func TestServer_AddNetwork(t *testing.T) { cidrs []string err error } + type getVPCIPv6CIDRsCall struct { + cidrs []string + err error + } type useExternalSNATCall struct { useExternalSNAT bool } @@ -93,9 +98,14 @@ func TestServer_AddNetwork(t *testing.T) { type fields struct { ipV4AddressByENIID map[string][]string + ipV6PrefixByENIID map[string][]string getVPCIPv4CIDRsCalls []getVPCIPv4CIDRsCall + getVPCIPv6CIDRsCalls []getVPCIPv6CIDRsCall useExternalSNATCalls []useExternalSNATCall getExcludeSNATCIDRsCalls []getExcludeSNATCIDRsCall + ipV4Enabled bool + ipV6Enabled bool + prefixDelegationEnabled bool } tests := []struct { name string @@ -104,7 +114,7 @@ func TestServer_AddNetwork(t *testing.T) { wantErr error }{ { - name: "successfully allocated IPAddress & use externalSNAT", + name: "successfully allocated IPv4Address & use externalSNAT", fields: fields{ ipV4AddressByENIID: map[string][]string{ "eni-1": {"192.168.1.100"}, @@ -119,17 +129,19 @@ func TestServer_AddNetwork(t *testing.T) { useExternalSNAT: true, }, }, + ipV4Enabled: true, + ipV6Enabled: false, }, want: &pb.AddNetworkReply{ Success: true, IPv4Addr: "192.168.1.100", DeviceNumber: int32(0), UseExternalSNAT: true, - VPCcidrs: []string{"10.10.0.0/16"}, + VPCV4Cidrs: []string{"10.10.0.0/16"}, }, }, { - name: "successfully allocated IPAddress & not use externalSNAT", + name: "successfully allocated IPv4Address & not use externalSNAT", fields: fields{ ipV4AddressByENIID: map[string][]string{ "eni-1": {"192.168.1.100"}, @@ -149,22 +161,78 @@ func TestServer_AddNetwork(t *testing.T) { snatExclusionCIDRs: []string{"10.12.0.0/16", "10.13.0.0/16"}, }, }, + ipV4Enabled: true, + ipV6Enabled: false, }, want: &pb.AddNetworkReply{ Success: true, IPv4Addr: "192.168.1.100", DeviceNumber: int32(0), UseExternalSNAT: false, - VPCcidrs: []string{"10.10.0.0/16", "10.12.0.0/16", "10.13.0.0/16"}, + VPCV4Cidrs: []string{"10.10.0.0/16", "10.12.0.0/16", "10.13.0.0/16"}, }, }, { - name: "failed allocated IPAddress ", + name: "failed allocating IPv4Address ", fields: fields{ ipV4AddressByENIID: map[string][]string{}, + ipV4Enabled: true, + ipV6Enabled: false, + }, + want: &pb.AddNetworkReply{ + Success: false, + DeviceNumber: int32(-1), + }, + }, + { + name: "successfully allocated IPv6Address in PD mode", + fields: fields{ + ipV6PrefixByENIID: map[string][]string{ + "eni-1": {"2001:db8::/64"}, + }, + getVPCIPv6CIDRsCalls: []getVPCIPv6CIDRsCall{ + { + cidrs: []string{"2001:db8::/56"}, + }, + }, + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: true, + }, + want: &pb.AddNetworkReply{ + Success: true, + IPv6Addr: "2001:db8::", + DeviceNumber: int32(0), + VPCV6Cidrs: []string{"2001:db8::/56"}, + }, + }, + { + name: "failed allocating IPv6Address - No IP addresses available", + fields: fields{ + ipV6PrefixByENIID: map[string][]string{}, + ipV4Enabled: true, + ipV6Enabled: false, + prefixDelegationEnabled: true, }, want: &pb.AddNetworkReply{ - Success: false, + Success: false, + DeviceNumber: int32(-1), + }, + }, + { + name: "failed allocating IPv6Address - PD disabled", + fields: fields{ + ipV6PrefixByENIID: map[string][]string{ + "eni-1": {"2001:db8::/64"}, + }, + ipV4Enabled: false, + ipV6Enabled: true, + prefixDelegationEnabled: false, + }, + want: &pb.AddNetworkReply{ + Success: false, + IPv6Addr: "", + DeviceNumber: int32(-1), }, }, } @@ -176,13 +244,16 @@ func TestServer_AddNetwork(t *testing.T) { for _, call := range tt.fields.getVPCIPv4CIDRsCalls { m.awsutils.EXPECT().GetVPCIPv4CIDRs().Return(call.cidrs, call.err) } + for _, call := range tt.fields.getVPCIPv6CIDRsCalls { + m.awsutils.EXPECT().GetVPCIPv6CIDRs().Return(call.cidrs, call.err) + } for _, call := range tt.fields.useExternalSNATCalls { m.network.EXPECT().UseExternalSNAT().Return(call.useExternalSNAT) } for _, call := range tt.fields.getExcludeSNATCIDRsCalls { m.network.EXPECT().GetExcludeSNATCIDRs().Return(call.snatExclusionCIDRs) } - ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, false) + ds := datastore.NewDataStore(log, datastore.NullCheckpoint{}, tt.fields.prefixDelegationEnabled) for eniID, ipv4Addresses := range tt.fields.ipV4AddressByENIID { ds.AddENI(eniID, 0, false, false, false) for _, ipv4Address := range ipv4Addresses { @@ -190,15 +261,25 @@ func TestServer_AddNetwork(t *testing.T) { ds.AddIPv4CidrToStore(eniID, ipv4Addr, false) } } + for eniID, ipv6Prefixes := range tt.fields.ipV6PrefixByENIID { + ds.AddENI(eniID, 0, false, false, false) + for _, ipv6Prefix := range ipv6Prefixes { + _, ipnet, _ := net.ParseCIDR(ipv6Prefix) + ds.AddIPv6CidrToStore(eniID, *ipnet, true) + } + } mockContext := &IPAMContext{ - awsClient: m.awsutils, - maxIPsPerENI: 14, - maxENI: 4, - warmENITarget: 1, - warmIPTarget: 3, - networkClient: m.network, - dataStore: ds, + awsClient: m.awsutils, + maxIPsPerENI: 14, + maxENI: 4, + warmENITarget: 1, + warmIPTarget: 3, + networkClient: m.network, + enableIPv4: tt.fields.ipV4Enabled, + enableIPv6: tt.fields.ipV6Enabled, + enablePrefixDelegation: tt.fields.prefixDelegationEnabled, + dataStore: ds, } s := &server{ diff --git a/pkg/networkutils/mocks/network_mocks.go b/pkg/networkutils/mocks/network_mocks.go index 2bb31bef5a..1d91e7d824 100644 --- a/pkg/networkutils/mocks/network_mocks.go +++ b/pkg/networkutils/mocks/network_mocks.go @@ -138,7 +138,7 @@ func (mr *MockNetworkAPIsMockRecorder) SetupENINetwork(arg0, arg1, arg2, arg3 in } // SetupHostNetwork mocks base method -func (m *MockNetworkAPIs) SetupHostNetwork(arg0 []string, arg1 string, arg2 *net.IP, arg3 bool) error { +func (m *MockNetworkAPIs) SetupHostNetwork(arg0 []string, arg1 string, arg2 *net.IP, arg3, arg4, arg5 bool) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SetupHostNetwork", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) @@ -146,13 +146,13 @@ func (m *MockNetworkAPIs) SetupHostNetwork(arg0 []string, arg1 string, arg2 *net } // SetupHostNetwork indicates an expected call of SetupHostNetwork -func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) SetupHostNetwork(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetupHostNetwork", reflect.TypeOf((*MockNetworkAPIs)(nil).SetupHostNetwork), arg0, arg1, arg2, arg3) } // UpdateHostIptablesRules mocks base method -func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, arg2 *net.IP) error { +func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, arg2 *net.IP, arg3, arg4 bool) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdateHostIptablesRules", arg0, arg1, arg2) ret0, _ := ret[0].(error) @@ -160,7 +160,7 @@ func (m *MockNetworkAPIs) UpdateHostIptablesRules(arg0 []string, arg1 string, ar } // UpdateHostIptablesRules indicates an expected call of UpdateHostIptablesRules -func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockNetworkAPIsMockRecorder) UpdateHostIptablesRules(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateHostIptablesRules", reflect.TypeOf((*MockNetworkAPIs)(nil).UpdateHostIptablesRules), arg0, arg1, arg2) } diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index 4947bb2ba8..9635a350b2 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -128,11 +128,12 @@ var log = logger.Get() // NetworkAPIs defines the host level and the ENI level network related operations type NetworkAPIs interface { // SetupNodeNetwork performs node level network configuration - SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool) error + SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool, + v4Enabled bool, v6Enabled bool) error // SetupENINetwork performs ENI level network configuration. Not needed on the primary ENI SetupENINetwork(eniIP string, mac string, deviceNumber int, subnetCIDR string) error // UpdateHostIptablesRules updates the nat table iptables rules on the host - UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error + UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, v6Enabled bool) error UseExternalSNAT() bool GetExcludeSNATCIDRs() []string GetRuleList() ([]netlink.Rule, error) @@ -153,7 +154,7 @@ type linuxNetwork struct { netLink netlinkwrapper.NetLink ns nswrapper.NS - newIptables func() (iptablesIface, error) + newIptables func(IPProtocol iptables.Protocol) (iptablesIface, error) mainENIMark uint32 procSys procsyswrapper.ProcSys } @@ -193,8 +194,8 @@ func New() NetworkAPIs { netLink: netlinkwrapper.NewNetLink(), ns: nswrapper.NewNS(), - newIptables: func() (iptablesIface, error) { - ipt, err := iptables.New() + newIptables: func(IPProtocol iptables.Protocol) (iptablesIface, error) { + ipt, err := iptables.NewWithProtocol(IPProtocol) return ipt, err }, procSys: procsyswrapper.NewProcSys(), @@ -224,13 +225,38 @@ func findPrimaryInterfaceName(primaryMAC string) (string, error) { return "", errors.New("no primary interface found") } +func (n *linuxNetwork) enableIPv6() (err error) { + //Make sure IPv6 is enabled on host interfaces + if err = n.procSys.Set(fmt.Sprintf("net/ipv6/conf/all/disable_ipv6"), "0"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on hostVeth interface") + } + } + //Make sure IPv6 forwarding is enabled on all host interfaces + if err = n.procSys.Set(fmt.Sprintf("net/ipv6/conf/all/forwarding"), "1"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 forwarding on all host interfaces") + } + } + + //Let's disable ipv6 forwarding on eth0 (Primary ENI). We will direct external v6 traffic via GW + if err = n.procSys.Set(fmt.Sprintf("net/ipv6/conf/eth0/forwarding"), "0"); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "setupVeth network: failed to disable IPv6 forwarding on eth0 interface") + } + } + return nil +} + // SetupHostNetwork performs node level network configuration -func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool) error { +func (n *linuxNetwork) SetupHostNetwork(vpcv4CIDRs []string, primaryMAC string, primaryAddr *net.IP, enablePodENI bool, + v4Enabled bool, v6Enabled bool) error { log.Info("Setting up host network... ") var err error primaryIntf := "eth0" - if n.nodePortSupportEnabled { + //RP Filter setting is only needed if IPv4 mode is enabled. + if v4Enabled && n.nodePortSupportEnabled { primaryIntf, err = findPrimaryInterfaceName(primaryMAC) if err != nil { return errors.Wrapf(err, "failed to SetupHostNetwork") @@ -266,16 +292,27 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr return errors.Wrapf(err, "setupHostNetwork: failed to set MTU to %d for %s", n.mtu, primaryIntf) } + ipFamily := unix.AF_INET + if v6Enabled { + ipFamily = unix.AF_INET6 + if err := n.enableIPv6(); err != nil { + return errors.Wrapf(err, "failed to enable IPv6") + } + } + // If node port support is enabled, add a rule that will force force marked traffic out of the main ENI. We then // add iptables rules below that will mark traffic that needs this special treatment. In particular NodePort // traffic always comes in via the main ENI but response traffic would go out of the pod's assigned ENI if we // didn't handle it specially. This is because the routing decision is done before the NodePort's DNAT is // reversed so, to the routing table, it looks like the traffic is pod traffic instead of NodePort traffic. + // Note: With v6 PD mode support, all the pods will be behind Primary ENI of the node and so we might not even need + // to mark the packets entering via Primary ENI for NodePort support. mainENIRule := n.netLink.NewRule() mainENIRule.Mark = int(n.mainENIMark) mainENIRule.Mask = int(n.mainENIMark) mainENIRule.Table = mainRoutingTable mainENIRule.Priority = hostRulePriority + mainENIRule.Family = ipFamily // If this is a restart, cleanup previous rule first err = n.netLink.RuleDel(mainENIRule) if err != nil && !containsNoSuchRule(err) { @@ -293,7 +330,9 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr // If we want per pod ENIs, we need to give pod ENIs veth bridges a lower priority that the local table, // or the rp_filter check will fail. - if enablePodENI { + // Note: Per Pod Security Group is not supported for V6 yet. So, cordoning off the PPSG rule (for now) + // with v4 specific check. + if v4Enabled && enablePodENI { localRule := n.netLink.NewRule() localRule.Table = localRouteTable localRule.Priority = localRulePriority @@ -310,37 +349,52 @@ func (n *linuxNetwork) SetupHostNetwork(vpcCIDRs []string, primaryMAC string, pr } } - return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr) + return n.updateHostIptablesRules(vpcv4CIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled) } // UpdateHostIptablesRules updates the NAT table rules based on the VPC CIDRs configuration -func (n *linuxNetwork) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error { - return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr) +func (n *linuxNetwork) UpdateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, + v6Enabled bool) error { + return n.updateHostIptablesRules(vpcCIDRs, primaryMAC, primaryAddr, v4Enabled, v6Enabled) } -func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP) error { +func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC string, primaryAddr *net.IP, v4Enabled bool, + v6Enabled bool) error { primaryIntf, err := findPrimaryInterfaceName(primaryMAC) if err != nil { return errors.Wrapf(err, "failed to SetupHostNetwork") } - ipt, err := n.newIptables() - if err != nil { - return errors.Wrap(err, "host network setup: failed to create iptables") - } - iptablesSNATRules, err := n.buildIptablesSNATRules(vpcCIDRs, primaryAddr, primaryIntf, ipt) - if err != nil { - return err - } - if err := n.updateIptablesRules(iptablesSNATRules, ipt); err != nil { - return err + + ipProtocol := iptables.ProtocolIPv4 + if v6Enabled{ + //Essentially a stub function for now in V6 mode. We will need it when we support v6 in secondary IP and + //custom networking modes. We don't need to install any SNAT rules in v6 mode and currently there is no need + //to mark packets entering via Primary ENI as all the pods in v6 mode will be behind primary ENI. Will have to + //start doing that once we start supporting custom networking mode in v6. + ipProtocol = iptables.ProtocolIPv6 } - iptablesConnmarkRules, err := n.buildIptablesConnmarkRules(vpcCIDRs, ipt) + ipt, err := n.newIptables(ipProtocol) if err != nil { - return err + return errors.Wrap(err, "host network setup: failed to create iptables") } - if err := n.updateIptablesRules(iptablesConnmarkRules, ipt); err != nil { - return err + + if v4Enabled { + iptablesSNATRules, err := n.buildIptablesSNATRules(vpcCIDRs, primaryAddr, primaryIntf, ipt) + if err != nil { + return err + } + if err := n.updateIptablesRules(iptablesSNATRules, ipt); err != nil { + return err + } + + iptablesConnmarkRules, err := n.buildIptablesConnmarkRules(vpcCIDRs, ipt) + if err != nil { + return err + } + if err := n.updateIptablesRules(iptablesConnmarkRules, ipt); err != nil { + return err + } } return nil } diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go index adc6e946d2..c174d10f4a 100644 --- a/pkg/networkutils/network_test.go +++ b/pkg/networkutils/network_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/coreos/go-iptables/iptables" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" @@ -150,7 +151,7 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) { mtu: testMTU, netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, } @@ -162,7 +163,7 @@ func TestSetupHostNetworkNodePortDisabled(t *testing.T) { mockNetLink.EXPECT().RuleDel(&mainENIRule) var vpcCIDRs []string - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) } @@ -232,19 +233,21 @@ func TestSetupHostNetworkNodePortEnabled(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, } + log.Debugf("mockIPtables.Dp state: ", mockIptables.dataplaneState) setupNetLinkMocks(ctrl, mockNetLink) + log.Debugf("After: mockIPtables.Dp state: ", mockIptables.dataplaneState) mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) var vpcCIDRs []string - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, map[string]map[string][][]string{ @@ -307,7 +310,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -318,7 +321,7 @@ func TestSetupHostNetworkWithExcludeSNATCIDRs(t *testing.T) { mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, map[string]map[string][][]string{ @@ -364,7 +367,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -389,7 +392,7 @@ func TestSetupHostNetworkCleansUpStaleSNATRules(t *testing.T) { _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, @@ -436,7 +439,7 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -462,7 +465,7 @@ func TestSetupHostNetworkWithDifferentVethPrefix(t *testing.T) { _ = mockIptables.Append("nat", "PREROUTING", "-m", "comment", "--comment", "AWS, CONNMARK", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, map[string]map[string][][]string{ @@ -508,7 +511,7 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -533,7 +536,7 @@ func TestSetupHostNetworkExternalNATCleanupConnmark(t *testing.T) { // remove exclusions vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, @@ -576,7 +579,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -601,7 +604,7 @@ func TestSetupHostNetworkExcludedSNATCIDRsIdempotent(t *testing.T) { // remove exclusions vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, @@ -647,7 +650,7 @@ func TestUpdateHostIptablesRules(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -669,7 +672,7 @@ func TestUpdateHostIptablesRules(t *testing.T) { _ = mockIptables.Append("mangle", "PREROUTING", "-m", "comment", "--comment", "AWS, primary ENI", "-i", "vlan+", "-j", "CONNMARK", "--restore-mark", "--mask", "0x80") vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) assert.Equal(t, map[string]map[string][][]string{ @@ -711,7 +714,7 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -721,7 +724,39 @@ func TestSetupHostNetworkMultipleCIDRs(t *testing.T) { mockProcSys.EXPECT().Set("net/ipv4/conf/lo/rp_filter", "2").Return(nil) vpcCIDRs := []string{"10.10.0.0/16", "10.11.0.0/16"} - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) + assert.NoError(t, err) +} + +func TestSetupHostNetworkWithIPv6Enabled(t *testing.T) { + ctrl, mockNetLink, _, mockNS, mockIptables, mockProcSys := setup(t) + defer ctrl.Finish() + + ln := &linuxNetwork{ + useExternalSNAT: false, + excludeSNATCIDRs: nil, + nodePortSupportEnabled: true, + shouldConfigureRpFilter: true, + mainENIMark: defaultConnmark, + mtu: testMTU, + vethPrefix: eniPrefix, + + netLink: mockNetLink, + ns: mockNS, + newIptables: func(iptables.Protocol) (iptablesIface, error) { + return mockIptables, nil + }, + procSys: mockProcSys, + } + + setupNetLinkMocks(ctrl, mockNetLink) + mockProcSys.EXPECT().Set("net/ipv6/conf/all/disable_ipv6", "0").Return(nil) + mockProcSys.EXPECT().Set("net/ipv6/conf/all/forwarding", "1").Return(nil) + mockProcSys.EXPECT().Set("net/ipv6/conf/eth0/forwarding", "0").Return(nil) + + + var vpcCIDRs []string + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, false, true) assert.NoError(t, err) } @@ -765,7 +800,7 @@ func TestSetupHostNetworkIgnoringRpFilterUpdate(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -773,7 +808,7 @@ func TestSetupHostNetworkIgnoringRpFilterUpdate(t *testing.T) { setupNetLinkMocks(ctrl, mockNetLink) var vpcCIDRs []string - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, true, false) assert.NoError(t, err) } @@ -791,7 +826,7 @@ func TestSetupHostNetworkUpdateLocalRule(t *testing.T) { netLink: mockNetLink, ns: mockNS, - newIptables: func() (iptablesIface, error) { + newIptables: func(iptables.Protocol) (iptablesIface, error) { return mockIptables, nil }, procSys: mockProcSys, @@ -802,7 +837,7 @@ func TestSetupHostNetworkUpdateLocalRule(t *testing.T) { mockNetLink.EXPECT() var vpcCIDRs []string - err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, true) + err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, true, true, false) assert.NoError(t, err) } diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 40625b7c9b..33974d7670 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1,546 +1,621 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: rpc.proto +// versions: +// protoc-gen-go v1.26.0 +// protoc v3.17.2 +// source: rpc/rpc.proto package rpc import ( - context "context" - fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type AddNetworkRequest struct { - ClientVersion string `protobuf:"bytes,8,opt,name=ClientVersion,proto3" json:"ClientVersion,omitempty"` - K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"` - K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` - K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=K8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` - ContainerID string `protobuf:"bytes,7,opt,name=ContainerID,proto3" json:"ContainerID,omitempty"` - IfName string `protobuf:"bytes,5,opt,name=IfName,proto3" json:"IfName,omitempty"` - NetworkName string `protobuf:"bytes,6,opt,name=NetworkName,proto3" json:"NetworkName,omitempty"` - Netns string `protobuf:"bytes,4,opt,name=Netns,proto3" json:"Netns,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *AddNetworkRequest) Reset() { *m = AddNetworkRequest{} } -func (m *AddNetworkRequest) String() string { return proto.CompactTextString(m) } -func (*AddNetworkRequest) ProtoMessage() {} -func (*AddNetworkRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{0} -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *AddNetworkRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_AddNetworkRequest.Unmarshal(m, b) -} -func (m *AddNetworkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_AddNetworkRequest.Marshal(b, m, deterministic) + ClientVersion string `protobuf:"bytes,8,opt,name=ClientVersion,proto3" json:"ClientVersion,omitempty"` + K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"` + K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=K8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` + ContainerID string `protobuf:"bytes,7,opt,name=ContainerID,proto3" json:"ContainerID,omitempty"` + IfName string `protobuf:"bytes,5,opt,name=IfName,proto3" json:"IfName,omitempty"` + NetworkName string `protobuf:"bytes,6,opt,name=NetworkName,proto3" json:"NetworkName,omitempty"` + Netns string `protobuf:"bytes,4,opt,name=Netns,proto3" json:"Netns,omitempty"` // next field: 9 } -func (m *AddNetworkRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_AddNetworkRequest.Merge(m, src) + +func (x *AddNetworkRequest) Reset() { + *x = AddNetworkRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_rpc_rpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *AddNetworkRequest) XXX_Size() int { - return xxx_messageInfo_AddNetworkRequest.Size(m) + +func (x *AddNetworkRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *AddNetworkRequest) XXX_DiscardUnknown() { - xxx_messageInfo_AddNetworkRequest.DiscardUnknown(m) + +func (*AddNetworkRequest) ProtoMessage() {} + +func (x *AddNetworkRequest) ProtoReflect() protoreflect.Message { + mi := &file_rpc_rpc_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_AddNetworkRequest proto.InternalMessageInfo +// Deprecated: Use AddNetworkRequest.ProtoReflect.Descriptor instead. +func (*AddNetworkRequest) Descriptor() ([]byte, []int) { + return file_rpc_rpc_proto_rawDescGZIP(), []int{0} +} -func (m *AddNetworkRequest) GetClientVersion() string { - if m != nil { - return m.ClientVersion +func (x *AddNetworkRequest) GetClientVersion() string { + if x != nil { + return x.ClientVersion } return "" } -func (m *AddNetworkRequest) GetK8S_POD_NAME() string { - if m != nil { - return m.K8S_POD_NAME +func (x *AddNetworkRequest) GetK8S_POD_NAME() string { + if x != nil { + return x.K8S_POD_NAME } return "" } -func (m *AddNetworkRequest) GetK8S_POD_NAMESPACE() string { - if m != nil { - return m.K8S_POD_NAMESPACE +func (x *AddNetworkRequest) GetK8S_POD_NAMESPACE() string { + if x != nil { + return x.K8S_POD_NAMESPACE } return "" } -func (m *AddNetworkRequest) GetK8S_POD_INFRA_CONTAINER_ID() string { - if m != nil { - return m.K8S_POD_INFRA_CONTAINER_ID +func (x *AddNetworkRequest) GetK8S_POD_INFRA_CONTAINER_ID() string { + if x != nil { + return x.K8S_POD_INFRA_CONTAINER_ID } return "" } -func (m *AddNetworkRequest) GetContainerID() string { - if m != nil { - return m.ContainerID +func (x *AddNetworkRequest) GetContainerID() string { + if x != nil { + return x.ContainerID } return "" } -func (m *AddNetworkRequest) GetIfName() string { - if m != nil { - return m.IfName +func (x *AddNetworkRequest) GetIfName() string { + if x != nil { + return x.IfName } return "" } -func (m *AddNetworkRequest) GetNetworkName() string { - if m != nil { - return m.NetworkName +func (x *AddNetworkRequest) GetNetworkName() string { + if x != nil { + return x.NetworkName } return "" } -func (m *AddNetworkRequest) GetNetns() string { - if m != nil { - return m.Netns +func (x *AddNetworkRequest) GetNetns() string { + if x != nil { + return x.Netns } return "" } type AddNetworkReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"` + IPv6Addr string `protobuf:"bytes,3,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"` DeviceNumber int32 `protobuf:"varint,4,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"` UseExternalSNAT bool `protobuf:"varint,5,opt,name=UseExternalSNAT,proto3" json:"UseExternalSNAT,omitempty"` - VPCcidrs []string `protobuf:"bytes,6,rep,name=VPCcidrs,proto3" json:"VPCcidrs,omitempty"` + VPCV4Cidrs []string `protobuf:"bytes,6,rep,name=VPCV4cidrs,proto3" json:"VPCV4cidrs,omitempty"` + VPCV6Cidrs []string `protobuf:"bytes,7,rep,name=VPCV6cidrs,proto3" json:"VPCV6cidrs,omitempty"` // start of pod-eni parameters - PodVlanId int32 `protobuf:"varint,7,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` - PodENIMAC string `protobuf:"bytes,8,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` - PodENISubnetGW string `protobuf:"bytes,9,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` - ParentIfIndex int32 `protobuf:"varint,10,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *AddNetworkReply) Reset() { *m = AddNetworkReply{} } -func (m *AddNetworkReply) String() string { return proto.CompactTextString(m) } -func (*AddNetworkReply) ProtoMessage() {} -func (*AddNetworkReply) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{1} + PodVlanId int32 `protobuf:"varint,8,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` + PodENIMAC string `protobuf:"bytes,9,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` + PodENISubnetGW string `protobuf:"bytes,10,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` + ParentIfIndex int32 `protobuf:"varint,11,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` // end of pod-eni parameters } -func (m *AddNetworkReply) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_AddNetworkReply.Unmarshal(m, b) -} -func (m *AddNetworkReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_AddNetworkReply.Marshal(b, m, deterministic) -} -func (m *AddNetworkReply) XXX_Merge(src proto.Message) { - xxx_messageInfo_AddNetworkReply.Merge(m, src) +func (x *AddNetworkReply) Reset() { + *x = AddNetworkReply{} + if protoimpl.UnsafeEnabled { + mi := &file_rpc_rpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *AddNetworkReply) XXX_Size() int { - return xxx_messageInfo_AddNetworkReply.Size(m) + +func (x *AddNetworkReply) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *AddNetworkReply) XXX_DiscardUnknown() { - xxx_messageInfo_AddNetworkReply.DiscardUnknown(m) + +func (*AddNetworkReply) ProtoMessage() {} + +func (x *AddNetworkReply) ProtoReflect() protoreflect.Message { + mi := &file_rpc_rpc_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_AddNetworkReply proto.InternalMessageInfo +// Deprecated: Use AddNetworkReply.ProtoReflect.Descriptor instead. +func (*AddNetworkReply) Descriptor() ([]byte, []int) { + return file_rpc_rpc_proto_rawDescGZIP(), []int{1} +} -func (m *AddNetworkReply) GetSuccess() bool { - if m != nil { - return m.Success +func (x *AddNetworkReply) GetSuccess() bool { + if x != nil { + return x.Success } return false } -func (m *AddNetworkReply) GetIPv4Addr() string { - if m != nil { - return m.IPv4Addr +func (x *AddNetworkReply) GetIPv4Addr() string { + if x != nil { + return x.IPv4Addr } return "" } -func (m *AddNetworkReply) GetDeviceNumber() int32 { - if m != nil { - return m.DeviceNumber +func (x *AddNetworkReply) GetIPv6Addr() string { + if x != nil { + return x.IPv6Addr + } + return "" +} + +func (x *AddNetworkReply) GetDeviceNumber() int32 { + if x != nil { + return x.DeviceNumber } return 0 } -func (m *AddNetworkReply) GetUseExternalSNAT() bool { - if m != nil { - return m.UseExternalSNAT +func (x *AddNetworkReply) GetUseExternalSNAT() bool { + if x != nil { + return x.UseExternalSNAT } return false } -func (m *AddNetworkReply) GetVPCcidrs() []string { - if m != nil { - return m.VPCcidrs +func (x *AddNetworkReply) GetVPCV4Cidrs() []string { + if x != nil { + return x.VPCV4Cidrs + } + return nil +} + +func (x *AddNetworkReply) GetVPCV6Cidrs() []string { + if x != nil { + return x.VPCV6Cidrs } return nil } -func (m *AddNetworkReply) GetPodVlanId() int32 { - if m != nil { - return m.PodVlanId +func (x *AddNetworkReply) GetPodVlanId() int32 { + if x != nil { + return x.PodVlanId } return 0 } -func (m *AddNetworkReply) GetPodENIMAC() string { - if m != nil { - return m.PodENIMAC +func (x *AddNetworkReply) GetPodENIMAC() string { + if x != nil { + return x.PodENIMAC } return "" } -func (m *AddNetworkReply) GetPodENISubnetGW() string { - if m != nil { - return m.PodENISubnetGW +func (x *AddNetworkReply) GetPodENISubnetGW() string { + if x != nil { + return x.PodENISubnetGW } return "" } -func (m *AddNetworkReply) GetParentIfIndex() int32 { - if m != nil { - return m.ParentIfIndex +func (x *AddNetworkReply) GetParentIfIndex() int32 { + if x != nil { + return x.ParentIfIndex } return 0 } type DelNetworkRequest struct { - ClientVersion string `protobuf:"bytes,9,opt,name=ClientVersion,proto3" json:"ClientVersion,omitempty"` - K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"` - K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` - K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=K8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` - Reason string `protobuf:"bytes,5,opt,name=Reason,proto3" json:"Reason,omitempty"` - ContainerID string `protobuf:"bytes,8,opt,name=ContainerID,proto3" json:"ContainerID,omitempty"` - IfName string `protobuf:"bytes,6,opt,name=IfName,proto3" json:"IfName,omitempty"` - NetworkName string `protobuf:"bytes,7,opt,name=NetworkName,proto3" json:"NetworkName,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DelNetworkRequest) Reset() { *m = DelNetworkRequest{} } -func (m *DelNetworkRequest) String() string { return proto.CompactTextString(m) } -func (*DelNetworkRequest) ProtoMessage() {} -func (*DelNetworkRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{2} -} + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields -func (m *DelNetworkRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DelNetworkRequest.Unmarshal(m, b) -} -func (m *DelNetworkRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DelNetworkRequest.Marshal(b, m, deterministic) + ClientVersion string `protobuf:"bytes,9,opt,name=ClientVersion,proto3" json:"ClientVersion,omitempty"` + K8S_POD_NAME string `protobuf:"bytes,1,opt,name=K8S_POD_NAME,json=K8SPODNAME,proto3" json:"K8S_POD_NAME,omitempty"` + K8S_POD_NAMESPACE string `protobuf:"bytes,2,opt,name=K8S_POD_NAMESPACE,json=K8SPODNAMESPACE,proto3" json:"K8S_POD_NAMESPACE,omitempty"` + K8S_POD_INFRA_CONTAINER_ID string `protobuf:"bytes,3,opt,name=K8S_POD_INFRA_CONTAINER_ID,json=K8SPODINFRACONTAINERID,proto3" json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"` + Reason string `protobuf:"bytes,5,opt,name=Reason,proto3" json:"Reason,omitempty"` + ContainerID string `protobuf:"bytes,8,opt,name=ContainerID,proto3" json:"ContainerID,omitempty"` + IfName string `protobuf:"bytes,6,opt,name=IfName,proto3" json:"IfName,omitempty"` + NetworkName string `protobuf:"bytes,7,opt,name=NetworkName,proto3" json:"NetworkName,omitempty"` // next field: 10 } -func (m *DelNetworkRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DelNetworkRequest.Merge(m, src) + +func (x *DelNetworkRequest) Reset() { + *x = DelNetworkRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_rpc_rpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } } -func (m *DelNetworkRequest) XXX_Size() int { - return xxx_messageInfo_DelNetworkRequest.Size(m) + +func (x *DelNetworkRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *DelNetworkRequest) XXX_DiscardUnknown() { - xxx_messageInfo_DelNetworkRequest.DiscardUnknown(m) + +func (*DelNetworkRequest) ProtoMessage() {} + +func (x *DelNetworkRequest) ProtoReflect() protoreflect.Message { + mi := &file_rpc_rpc_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -var xxx_messageInfo_DelNetworkRequest proto.InternalMessageInfo +// Deprecated: Use DelNetworkRequest.ProtoReflect.Descriptor instead. +func (*DelNetworkRequest) Descriptor() ([]byte, []int) { + return file_rpc_rpc_proto_rawDescGZIP(), []int{2} +} -func (m *DelNetworkRequest) GetClientVersion() string { - if m != nil { - return m.ClientVersion +func (x *DelNetworkRequest) GetClientVersion() string { + if x != nil { + return x.ClientVersion } return "" } -func (m *DelNetworkRequest) GetK8S_POD_NAME() string { - if m != nil { - return m.K8S_POD_NAME +func (x *DelNetworkRequest) GetK8S_POD_NAME() string { + if x != nil { + return x.K8S_POD_NAME } return "" } -func (m *DelNetworkRequest) GetK8S_POD_NAMESPACE() string { - if m != nil { - return m.K8S_POD_NAMESPACE +func (x *DelNetworkRequest) GetK8S_POD_NAMESPACE() string { + if x != nil { + return x.K8S_POD_NAMESPACE } return "" } -func (m *DelNetworkRequest) GetK8S_POD_INFRA_CONTAINER_ID() string { - if m != nil { - return m.K8S_POD_INFRA_CONTAINER_ID +func (x *DelNetworkRequest) GetK8S_POD_INFRA_CONTAINER_ID() string { + if x != nil { + return x.K8S_POD_INFRA_CONTAINER_ID } return "" } -func (m *DelNetworkRequest) GetReason() string { - if m != nil { - return m.Reason +func (x *DelNetworkRequest) GetReason() string { + if x != nil { + return x.Reason } return "" } -func (m *DelNetworkRequest) GetContainerID() string { - if m != nil { - return m.ContainerID +func (x *DelNetworkRequest) GetContainerID() string { + if x != nil { + return x.ContainerID } return "" } -func (m *DelNetworkRequest) GetIfName() string { - if m != nil { - return m.IfName +func (x *DelNetworkRequest) GetIfName() string { + if x != nil { + return x.IfName } return "" } -func (m *DelNetworkRequest) GetNetworkName() string { - if m != nil { - return m.NetworkName +func (x *DelNetworkRequest) GetNetworkName() string { + if x != nil { + return x.NetworkName } return "" } type DelNetworkReply struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"` DeviceNumber int32 `protobuf:"varint,3,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"` // start of pod-eni parameters - PodVlanId int32 `protobuf:"varint,4,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DelNetworkReply) Reset() { *m = DelNetworkReply{} } -func (m *DelNetworkReply) String() string { return proto.CompactTextString(m) } -func (*DelNetworkReply) ProtoMessage() {} -func (*DelNetworkReply) Descriptor() ([]byte, []int) { - return fileDescriptor_77a6da22d6a3feb1, []int{3} -} - -func (m *DelNetworkReply) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DelNetworkReply.Unmarshal(m, b) -} -func (m *DelNetworkReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DelNetworkReply.Marshal(b, m, deterministic) -} -func (m *DelNetworkReply) XXX_Merge(src proto.Message) { - xxx_messageInfo_DelNetworkReply.Merge(m, src) + PodVlanId int32 `protobuf:"varint,4,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` // end of pod-eni parameters } -func (m *DelNetworkReply) XXX_Size() int { - return xxx_messageInfo_DelNetworkReply.Size(m) -} -func (m *DelNetworkReply) XXX_DiscardUnknown() { - xxx_messageInfo_DelNetworkReply.DiscardUnknown(m) -} - -var xxx_messageInfo_DelNetworkReply proto.InternalMessageInfo -func (m *DelNetworkReply) GetSuccess() bool { - if m != nil { - return m.Success +func (x *DelNetworkReply) Reset() { + *x = DelNetworkReply{} + if protoimpl.UnsafeEnabled { + mi := &file_rpc_rpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return false } -func (m *DelNetworkReply) GetIPv4Addr() string { - if m != nil { - return m.IPv4Addr - } - return "" +func (x *DelNetworkReply) String() string { + return protoimpl.X.MessageStringOf(x) } -func (m *DelNetworkReply) GetDeviceNumber() int32 { - if m != nil { - return m.DeviceNumber - } - return 0 -} +func (*DelNetworkReply) ProtoMessage() {} -func (m *DelNetworkReply) GetPodVlanId() int32 { - if m != nil { - return m.PodVlanId +func (x *DelNetworkReply) ProtoReflect() protoreflect.Message { + mi := &file_rpc_rpc_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return 0 -} - -func init() { - proto.RegisterType((*AddNetworkRequest)(nil), "rpc.AddNetworkRequest") - proto.RegisterType((*AddNetworkReply)(nil), "rpc.AddNetworkReply") - proto.RegisterType((*DelNetworkRequest)(nil), "rpc.DelNetworkRequest") - proto.RegisterType((*DelNetworkReply)(nil), "rpc.DelNetworkReply") -} - -func init() { - proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) -} - -var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 507 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xcc, 0x94, 0xc1, 0x6e, 0xd3, 0x4e, - 0x10, 0xc6, 0x9b, 0xa4, 0x71, 0x92, 0xf9, 0xf7, 0x4f, 0x94, 0x55, 0x15, 0xad, 0x22, 0x0e, 0x91, - 0x85, 0x50, 0xc5, 0xa1, 0x07, 0xe0, 0x50, 0x21, 0x2e, 0xc6, 0x36, 0x68, 0x55, 0x75, 0x63, 0xd9, - 0x25, 0x1c, 0x23, 0xc7, 0x9e, 0x4a, 0x56, 0xdd, 0xb5, 0x59, 0x3b, 0xa5, 0x7d, 0x03, 0x78, 0x20, - 0xc4, 0x81, 0x97, 0x43, 0x5e, 0x3b, 0xb5, 0xe3, 0xa0, 0x72, 0xe1, 0xc0, 0x71, 0xbe, 0xf9, 0x3e, - 0x8d, 0x66, 0xfc, 0xf3, 0xc2, 0x48, 0xa6, 0xc1, 0x69, 0x2a, 0x93, 0x3c, 0x21, 0x3d, 0x99, 0x06, - 0xfa, 0xf7, 0x2e, 0x4c, 0x8c, 0x30, 0xe4, 0x98, 0x7f, 0x49, 0xe4, 0xb5, 0x8b, 0x9f, 0x37, 0x98, - 0xe5, 0xe4, 0x19, 0xfc, 0x6f, 0xc6, 0x11, 0x8a, 0x7c, 0x89, 0x32, 0x8b, 0x12, 0x41, 0x87, 0xf3, - 0xce, 0xc9, 0xc8, 0xdd, 0x15, 0xc9, 0x1c, 0x8e, 0xce, 0xcf, 0xbc, 0x95, 0xb3, 0xb0, 0x56, 0xdc, - 0xb8, 0xb0, 0x69, 0x47, 0x99, 0xe0, 0xfc, 0xcc, 0x73, 0x16, 0x56, 0xa1, 0x90, 0x17, 0x30, 0x69, - 0x3a, 0x3c, 0xc7, 0x30, 0x6d, 0xda, 0x55, 0xb6, 0x71, 0x6d, 0x53, 0x32, 0x79, 0x03, 0xb3, 0xad, - 0x97, 0xf1, 0xf7, 0xae, 0xb1, 0x32, 0x17, 0xfc, 0xd2, 0x60, 0xdc, 0x76, 0x57, 0xcc, 0xa2, 0x3d, - 0x15, 0x9a, 0x96, 0x21, 0xd5, 0x7f, 0x68, 0x33, 0x8b, 0xcc, 0xe1, 0x3f, 0x33, 0x11, 0xb9, 0x1f, - 0x09, 0x94, 0xcc, 0xa2, 0x03, 0x65, 0x6e, 0x4a, 0x64, 0x0a, 0x1a, 0xbb, 0xe2, 0xfe, 0x0d, 0xd2, - 0xbe, 0x6a, 0x56, 0x55, 0x91, 0xac, 0x76, 0x57, 0x4d, 0xad, 0x4c, 0x36, 0x24, 0x72, 0x0c, 0x7d, - 0x8e, 0xb9, 0xc8, 0xe8, 0xa1, 0xea, 0x95, 0x85, 0xfe, 0xb3, 0x0b, 0xe3, 0xe6, 0xdd, 0xd2, 0xf8, - 0x9e, 0x50, 0x18, 0x78, 0x9b, 0x20, 0xc0, 0x2c, 0x53, 0xa7, 0x18, 0xba, 0xdb, 0x92, 0xcc, 0x60, - 0xc8, 0x9c, 0xdb, 0xd7, 0x46, 0x18, 0xca, 0x6a, 0xfd, 0x87, 0x9a, 0xe8, 0x70, 0x64, 0xe1, 0x6d, - 0x14, 0x20, 0xdf, 0xdc, 0xac, 0x51, 0xaa, 0x31, 0x7d, 0x77, 0x47, 0x23, 0x27, 0x30, 0xfe, 0x98, - 0xa1, 0x7d, 0x97, 0xa3, 0x14, 0x7e, 0xec, 0x71, 0xe3, 0x52, 0xad, 0x31, 0x74, 0xdb, 0x72, 0x31, - 0x69, 0xe9, 0x98, 0x41, 0x14, 0xca, 0x8c, 0x6a, 0xf3, 0x5e, 0x31, 0x69, 0x5b, 0x93, 0xa7, 0x30, - 0x72, 0x92, 0x70, 0x19, 0xfb, 0x82, 0x85, 0xea, 0x46, 0x7d, 0xb7, 0x16, 0xaa, 0xae, 0xcd, 0xd9, - 0x85, 0x61, 0x56, 0xdf, 0xbb, 0x16, 0xc8, 0x73, 0x78, 0x52, 0x16, 0xde, 0x66, 0x2d, 0x30, 0xff, - 0xf0, 0x89, 0x8e, 0x94, 0xa5, 0xa5, 0x16, 0xe4, 0x38, 0xbe, 0x44, 0x91, 0xb3, 0x2b, 0x26, 0x42, - 0xbc, 0xa3, 0xa0, 0xe6, 0xec, 0x8a, 0xfa, 0x8f, 0x2e, 0x4c, 0x2c, 0x8c, 0xff, 0x44, 0xdd, 0xe8, - 0xdf, 0xa6, 0x6e, 0x0a, 0x9a, 0x8b, 0x7e, 0x96, 0x88, 0x2d, 0x53, 0x65, 0xd5, 0xa6, 0x71, 0xf8, - 0x18, 0x8d, 0xda, 0x63, 0x34, 0x0e, 0xf6, 0x68, 0xd4, 0xbf, 0x75, 0x60, 0xdc, 0xbc, 0xdc, 0xdf, - 0xe3, 0xae, 0xf7, 0x1b, 0xee, 0x76, 0x88, 0x39, 0x6c, 0x11, 0xf3, 0xf2, 0x6b, 0x07, 0xc0, 0xe4, - 0xec, 0x9d, 0x1f, 0x5c, 0xa3, 0x08, 0xc9, 0x5b, 0x80, 0xfa, 0x8f, 0x20, 0xd3, 0xd3, 0xe2, 0xa5, - 0xd9, 0x7b, 0x5a, 0x66, 0xc7, 0x7b, 0x7a, 0x1a, 0xdf, 0xeb, 0x07, 0x45, 0xba, 0xde, 0xab, 0x4a, - 0xef, 0x21, 0x52, 0xa5, 0x5b, 0x07, 0xd0, 0x0f, 0xd6, 0x9a, 0x7a, 0xd2, 0x5e, 0xfd, 0x0a, 0x00, - 0x00, 0xff, 0xff, 0xeb, 0x4a, 0x98, 0xac, 0xdf, 0x04, 0x00, 0x00, -} - -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConnInterface - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion6 - -// CNIBackendClient is the client API for CNIBackend service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type CNIBackendClient interface { - AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) - DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) -} - -type cNIBackendClient struct { - cc grpc.ClientConnInterface + return mi.MessageOf(x) } -func NewCNIBackendClient(cc grpc.ClientConnInterface) CNIBackendClient { - return &cNIBackendClient{cc} +// Deprecated: Use DelNetworkReply.ProtoReflect.Descriptor instead. +func (*DelNetworkReply) Descriptor() ([]byte, []int) { + return file_rpc_rpc_proto_rawDescGZIP(), []int{3} } -func (c *cNIBackendClient) AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) { - out := new(AddNetworkReply) - err := c.cc.Invoke(ctx, "/rpc.CNIBackend/AddNetwork", in, out, opts...) - if err != nil { - return nil, err +func (x *DelNetworkReply) GetSuccess() bool { + if x != nil { + return x.Success } - return out, nil + return false } -func (c *cNIBackendClient) DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) { - out := new(DelNetworkReply) - err := c.cc.Invoke(ctx, "/rpc.CNIBackend/DelNetwork", in, out, opts...) - if err != nil { - return nil, err +func (x *DelNetworkReply) GetIPv4Addr() string { + if x != nil { + return x.IPv4Addr } - return out, nil -} - -// CNIBackendServer is the server API for CNIBackend service. -type CNIBackendServer interface { - AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) - DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) -} - -// UnimplementedCNIBackendServer can be embedded to have forward compatible implementations. -type UnimplementedCNIBackendServer struct { -} - -func (*UnimplementedCNIBackendServer) AddNetwork(ctx context.Context, req *AddNetworkRequest) (*AddNetworkReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method AddNetwork not implemented") -} -func (*UnimplementedCNIBackendServer) DelNetwork(ctx context.Context, req *DelNetworkRequest) (*DelNetworkReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DelNetwork not implemented") -} - -func RegisterCNIBackendServer(s *grpc.Server, srv CNIBackendServer) { - s.RegisterService(&_CNIBackend_serviceDesc, srv) + return "" } -func _CNIBackend_AddNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AddNetworkRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CNIBackendServer).AddNetwork(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/rpc.CNIBackend/AddNetwork", +func (x *DelNetworkReply) GetDeviceNumber() int32 { + if x != nil { + return x.DeviceNumber } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CNIBackendServer).AddNetwork(ctx, req.(*AddNetworkRequest)) - } - return interceptor(ctx, in, info, handler) + return 0 } -func _CNIBackend_DelNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DelNetworkRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CNIBackendServer).DelNetwork(ctx, in) +func (x *DelNetworkReply) GetPodVlanId() int32 { + if x != nil { + return x.PodVlanId } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/rpc.CNIBackend/DelNetwork", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CNIBackendServer).DelNetwork(ctx, req.(*DelNetworkRequest)) - } - return interceptor(ctx, in, info, handler) + return 0 } -var _CNIBackend_serviceDesc = grpc.ServiceDesc{ - ServiceName: "rpc.CNIBackend", - HandlerType: (*CNIBackendServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "AddNetwork", - Handler: _CNIBackend_AddNetwork_Handler, - }, - { - MethodName: "DelNetwork", - Handler: _CNIBackend_DelNetwork_Handler, +var File_rpc_rpc_proto protoreflect.FileDescriptor + +var file_rpc_rpc_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x03, 0x72, 0x70, 0x63, 0x22, 0xb5, 0x02, 0x0a, 0x11, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, + 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, + 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, + 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, + 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x12, 0x3a, + 0x0a, 0x1a, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x5f, + 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x43, + 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, + 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, + 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0xfb, 0x02, 0x0a, + 0x0f, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, + 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, + 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, + 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, + 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, + 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, + 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x34, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x34, 0x63, 0x69, 0x64, 0x72, 0x73, + 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x36, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x07, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x36, 0x63, 0x69, 0x64, 0x72, 0x73, + 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1c, + 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x18, 0x09, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x12, 0x26, 0x0a, 0x0e, + 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x47, 0x57, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, + 0x65, 0x74, 0x47, 0x57, 0x12, 0x24, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x66, + 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x50, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x49, 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0xb7, 0x02, 0x0a, 0x11, 0x44, + 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, + 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, + 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, + 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, + 0x50, 0x41, 0x43, 0x45, 0x12, 0x3a, 0x0a, 0x1a, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, + 0x49, 0x4e, 0x46, 0x52, 0x41, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, + 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, + 0x49, 0x4e, 0x46, 0x52, 0x41, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, + 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x66, + 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, + 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, + 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, + 0x32, 0x88, 0x01, 0x0a, 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, + 0x3c, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, + 0x0a, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x72, + 0x70, 0x63, 0x2f, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_rpc_rpc_proto_rawDescOnce sync.Once + file_rpc_rpc_proto_rawDescData = file_rpc_rpc_proto_rawDesc +) + +func file_rpc_rpc_proto_rawDescGZIP() []byte { + file_rpc_rpc_proto_rawDescOnce.Do(func() { + file_rpc_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_rpc_rpc_proto_rawDescData) + }) + return file_rpc_rpc_proto_rawDescData +} + +var file_rpc_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_rpc_rpc_proto_goTypes = []interface{}{ + (*AddNetworkRequest)(nil), // 0: rpc.AddNetworkRequest + (*AddNetworkReply)(nil), // 1: rpc.AddNetworkReply + (*DelNetworkRequest)(nil), // 2: rpc.DelNetworkRequest + (*DelNetworkReply)(nil), // 3: rpc.DelNetworkReply +} +var file_rpc_rpc_proto_depIdxs = []int32{ + 0, // 0: rpc.CNIBackend.AddNetwork:input_type -> rpc.AddNetworkRequest + 2, // 1: rpc.CNIBackend.DelNetwork:input_type -> rpc.DelNetworkRequest + 1, // 2: rpc.CNIBackend.AddNetwork:output_type -> rpc.AddNetworkReply + 3, // 3: rpc.CNIBackend.DelNetwork:output_type -> rpc.DelNetworkReply + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_rpc_rpc_proto_init() } +func file_rpc_rpc_proto_init() { + if File_rpc_rpc_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_rpc_rpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AddNetworkRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rpc_rpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AddNetworkReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rpc_rpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DelNetworkRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_rpc_rpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DelNetworkReply); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_rpc_rpc_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "rpc.proto", + GoTypes: file_rpc_rpc_proto_goTypes, + DependencyIndexes: file_rpc_rpc_proto_depIdxs, + MessageInfos: file_rpc_rpc_proto_msgTypes, + }.Build() + File_rpc_rpc_proto = out.File + file_rpc_rpc_proto_rawDesc = nil + file_rpc_rpc_proto_goTypes = nil + file_rpc_rpc_proto_depIdxs = nil } diff --git a/rpc/rpc.proto b/rpc/rpc.proto index d35719c278..930bed18e3 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package rpc; +option go_package = "rpc/;rpc"; + // The service definition. service CNIBackend { rpc AddNetwork (AddNetworkRequest) returns (AddNetworkReply) {} @@ -23,18 +25,21 @@ message AddNetworkRequest { message AddNetworkReply { bool Success = 1; string IPv4Addr = 2; + string IPv6Addr = 3; int32 DeviceNumber = 4; bool UseExternalSNAT = 5; - repeated string VPCcidrs = 6; + repeated string VPCV4cidrs = 6; + repeated string VPCV6cidrs = 7; + // start of pod-eni parameters - int32 PodVlanId = 7; - string PodENIMAC = 8; - string PodENISubnetGW = 9; - int32 ParentIfIndex = 10; + int32 PodVlanId = 8; + string PodENIMAC = 9; + string PodENISubnetGW = 10; + int32 ParentIfIndex = 11; // end of pod-eni parameters - // next field: 11 + // next field: 12 } message DelNetworkRequest { diff --git a/rpc/rpc_grpc.pb.go b/rpc/rpc_grpc.pb.go new file mode 100644 index 0000000000..2c745f419d --- /dev/null +++ b/rpc/rpc_grpc.pb.go @@ -0,0 +1,136 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. + +package rpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion6 + +// CNIBackendClient is the client API for CNIBackend service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type CNIBackendClient interface { + AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) + DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) +} + +type cNIBackendClient struct { + cc grpc.ClientConnInterface +} + +func NewCNIBackendClient(cc grpc.ClientConnInterface) CNIBackendClient { + return &cNIBackendClient{cc} +} + +func (c *cNIBackendClient) AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) { + out := new(AddNetworkReply) + err := c.cc.Invoke(ctx, "/rpc.CNIBackend/AddNetwork", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cNIBackendClient) DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) { + out := new(DelNetworkReply) + err := c.cc.Invoke(ctx, "/rpc.CNIBackend/DelNetwork", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CNIBackendServer is the server API for CNIBackend service. +// All implementations must embed UnimplementedCNIBackendServer +// for forward compatibility +type CNIBackendServer interface { + AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) + DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) +} + +// UnimplementedCNIBackendServer must be embedded to have forward compatible implementations. +type UnimplementedCNIBackendServer struct { +} + +func (UnimplementedCNIBackendServer) AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddNetwork not implemented") +} +func (UnimplementedCNIBackendServer) DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DelNetwork not implemented") +} +func (UnimplementedCNIBackendServer) mustEmbedUnimplementedCNIBackendServer() {} + +// UnsafeCNIBackendServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to CNIBackendServer will +// result in compilation errors. +type UnsafeCNIBackendServer interface { + mustEmbedUnimplementedCNIBackendServer() +} + +func RegisterCNIBackendServer(s *grpc.Server, srv CNIBackendServer) { + s.RegisterService(&CNIBackend_ServiceDesc, srv) +} + +func _CNIBackend_AddNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AddNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CNIBackendServer).AddNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/rpc.CNIBackend/AddNetwork", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CNIBackendServer).AddNetwork(ctx, req.(*AddNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CNIBackend_DelNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DelNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CNIBackendServer).DelNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/rpc.CNIBackend/DelNetwork", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CNIBackendServer).DelNetwork(ctx, req.(*DelNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// CNIBackend_ServiceDesc is the grpc.ServiceDesc for CNIBackend service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var CNIBackend_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "rpc.CNIBackend", + HandlerType: (*CNIBackendServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "AddNetwork", + Handler: _CNIBackend_AddNetwork_Handler, + }, + { + MethodName: "DelNetwork", + Handler: _CNIBackend_DelNetwork_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "rpc/rpc.proto", +} diff --git a/scripts/dockerfiles/Dockerfile.release b/scripts/dockerfiles/Dockerfile.release index 42957c68a5..dc3ea4357b 100644 --- a/scripts/dockerfiles/Dockerfile.release +++ b/scripts/dockerfiles/Dockerfile.release @@ -31,9 +31,11 @@ COPY --from=builder /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni \ /go/src/github.com/aws/amazon-vpc-cni-k8s/loopback \ /go/src/github.com/aws/amazon-vpc-cni-k8s/portmap \ /go/src/github.com/aws/amazon-vpc-cni-k8s/bandwidth \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/host-local \ /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-cni-support.sh \ /go/src/github.com/aws/amazon-vpc-cni-k8s/aws-k8s-agent \ /go/src/github.com/aws/amazon-vpc-cni-k8s/grpc-health-probe \ + /go/src/github.com/aws/amazon-vpc-cni-k8s/egress-v4-cni \ /go/src/github.com/aws/amazon-vpc-cni-k8s/scripts/entrypoint.sh /app/ ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/scripts/dockerfiles/Dockerfile.test b/scripts/dockerfiles/Dockerfile.test index 0c50717436..2f518d60a1 100644 --- a/scripts/dockerfiles/Dockerfile.test +++ b/scripts/dockerfiles/Dockerfile.test @@ -17,7 +17,6 @@ RUN go get -u golang.org/x/tools/cmd/goimports # go.mod and go.sum go into their own layers. COPY go.mod . COPY go.sum . -COPY vendor/ vendor/ # This ensures `go mod download` happens only when go.mod and go.sum change. RUN go mod download diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index 3c88a86e7c..d9c5a2fa26 100755 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -95,6 +95,8 @@ AWS_VPC_K8S_CNI_VETHPREFIX=${AWS_VPC_K8S_CNI_VETHPREFIX:-"eni"} AWS_VPC_ENI_MTU=${AWS_VPC_ENI_MTU:-"9001"} AWS_VPC_K8S_PLUGIN_LOG_FILE=${AWS_VPC_K8S_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/plugin.log"} AWS_VPC_K8S_PLUGIN_LOG_LEVEL=${AWS_VPC_K8S_PLUGIN_LOG_LEVEL:-"Debug"} +AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE=${AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE:-"/var/log/aws-routed-eni/egress-v4-plugin.log"} +NODE_IP=${NODE_IP:=""} AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER=${AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER:-"true"} ENABLE_PREFIX_DELEGATION=${ENABLE_PREFIX_DELEGATION:-"false"} @@ -117,19 +119,33 @@ wait_for_ipam() { done } +#NodeIP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) +get_node_primary_v4_address() { + while : + do + NODE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) + if [[ "${NODE_IP}" != "" ]]; then + return 0 + fi + # We sleep for 1 second between each retry + sleep 1 + done +} + # If there is no init container, copy the required files if [[ "$AWS_VPC_K8S_CNI_CONFIGURE_RPFILTER" != "false" ]]; then # Copy files log_in_json info "Copying CNI plugin binaries ... " - PLUGIN_BINS="loopback portmap bandwidth aws-cni-support.sh" + PLUGIN_BINS="loopback portmap bandwidth host-local aws-cni-support.sh" for b in $PLUGIN_BINS; do # Install the binary install "$b" "$HOST_CNI_BIN_PATH" done fi -log_in_json info "Install CNI binary.." +log_in_json info "Install CNI binaries.." install aws-cni "$HOST_CNI_BIN_PATH" +install egress-v4-cni "$HOST_CNI_BIN_PATH" log_in_json info "Starting IPAM daemon in the background ... " ./aws-k8s-agent | tee -i "$AGENT_LOG_PATH" 2>&1 & @@ -142,6 +158,7 @@ if ! wait_for_ipam; then exit 1 fi +get_node_primary_v4_address log_in_json info "Copying config file ... " # modify the static config to populate it with the env vars @@ -150,6 +167,9 @@ sed \ -e s~__MTU__~"${AWS_VPC_ENI_MTU}"~g \ -e s~__PLUGINLOGFILE__~"${AWS_VPC_K8S_PLUGIN_LOG_FILE}"~g \ -e s~__PLUGINLOGLEVEL__~"${AWS_VPC_K8S_PLUGIN_LOG_LEVEL}"~g \ + -e s~__EGRESSV4PLUGINLOGFILE__~"${AWS_VPC_K8S_EGRESS_V4_PLUGIN_LOG_FILE}"~g \ + -e s~__EGRESSV4PLUGINENABLED__~"${ENABLE_IPv6}"~g \ + -e s~__NODEIP__~"${NODE_IP}"~g \ 10-aws.conflist > "$HOST_CNI_CONFDIR_PATH/10-aws.conflist" log_in_json info "Successfully copied CNI plugin binary and config file." From af8a0c8dfe278f4726d5d9211f1dd4a6dfaa0486 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Tue, 7 Sep 2021 16:42:52 -0700 Subject: [PATCH 2/7] Addressed CR comments --- cmd/egress-v4-cni-plugin/cni.go | 20 +- cmd/egress-v4-cni-plugin/snat/snat.go | 11 - cmd/routed-eni-cni-plugin/cni.go | 6 +- cmd/routed-eni-cni-plugin/driver/driver.go | 18 +- pkg/awsutils/awsutils.go | 16 +- pkg/awsutils/awsutils_test.go | 12 +- pkg/awsutils/imds.go | 21 +- pkg/awsutils/imds_test.go | 32 ++- pkg/ec2wrapper/client.go | 4 +- pkg/ipamd/datastore/data_store.go | 56 +++-- pkg/ipamd/datastore/data_store_test.go | 4 +- pkg/ipamd/ipamd.go | 11 +- pkg/ipamd/ipamd_test.go | 7 +- pkg/ipamd/rpc_handler.go | 5 +- pkg/ipamd/rpc_handler_test.go | 6 +- pkg/networkutils/network.go | 31 ++- pkg/networkutils/network_test.go | 20 +- rpc/rpc.pb.go | 274 ++++++++++----------- rpc/rpc.proto | 13 +- 19 files changed, 301 insertions(+), 266 deletions(-) diff --git a/cmd/egress-v4-cni-plugin/cni.go b/cmd/egress-v4-cni-plugin/cni.go index b8c31d4bbb..49b866b5fa 100644 --- a/cmd/egress-v4-cni-plugin/cni.go +++ b/cmd/egress-v4-cni-plugin/cni.go @@ -30,7 +30,6 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils" "github.com/vishvananda/netlink" - "github.com/containernetworking/plugins/pkg/utils/sysctl" "github.com/aws/amazon-vpc-cni-k8s/cmd/egress-v4-cni-plugin/snat" ) @@ -167,7 +166,7 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu for _, r := range []netlink.Route{ { - LinkIndex: contVeth.Index, //TODO - Should be default route + LinkIndex: contVeth.Index, Dst: &net.IPNet{ IP: ipc.Gateway, Mask: net.CIDRMask(addrBits, addrBits), @@ -191,17 +190,6 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu } } } - - //Disable IPv6 on this interface - _, err = sysctl.Sysctl("net/ipv6/conf/"+ifName+"/disable_ipv6", "1") - if err != nil { - return fmt.Errorf("failed to disable IPv6 for interface: %v", err) - } - //Block traffic directed to 169.254.172.0/22 from the Pod - err = snat.SetupRuleToBlockNodeLocalV4Access() - if err != nil { - return err - } return nil }) if err != nil { @@ -343,7 +331,7 @@ func cmdAdd(args *skel.CmdArgs) error { //Copy interfaces over to result, but not IPs. result.Interfaces = append(result.Interfaces, tmpResult.Interfaces...) //Note: Useful for debug, will do away with the below log prior to release - for _,v := range result.IPs { + for _, v := range result.IPs { log.Debugf("Interface Name: %v; IP: %s", v.Interface, v.Address) } @@ -388,7 +376,7 @@ func cmdDel(args *skel.CmdArgs) error { } //Retrieve IP addresses assigned to the interface - addrs, err := netlink.AddrList(iface, netlink.FAMILY_ALL) + addrs, err := netlink.AddrList(iface, netlink.FAMILY_V4) if err != nil { return fmt.Errorf("failed to get IP addresses for %q: %v", netConf.IfName, err) } @@ -410,10 +398,8 @@ func cmdDel(args *skel.CmdArgs) error { }) //DEL should be best effort. We should clean up as much as we can and avoid returning error - //CNI Spec: TODO if err != nil { log.Debugf("DEL: Executing in container ns errored out, returning", err) - return err } } diff --git a/cmd/egress-v4-cni-plugin/snat/snat.go b/cmd/egress-v4-cni-plugin/snat/snat.go index b6a148a360..3be955e1e3 100644 --- a/cmd/egress-v4-cni-plugin/snat/snat.go +++ b/cmd/egress-v4-cni-plugin/snat/snat.go @@ -42,17 +42,6 @@ func iptRules4(target, src net.IP, chain, comment string, useRandomFully bool) [ return rules } -//Setup a rule to block egress traffic directed to 169.254.172.0/22 from the Pod -func SetupRuleToBlockNodeLocalV4Access() error{ - ipt, _ := iptables.NewWithProtocol(iptables.ProtocolIPv4) - if err := ipt.AppendUnique("filter", "OUTPUT", "-d", "169.254.172.0/22", "-m", "comment", - "--comment", "Block Node Local Pod access via IPv4", "-j", "DROP"); err != nil { - return fmt.Errorf("failed adding v4 drop route: %v", err) - } - - return nil -} - // Snat4 SNATs IPv4 connections from `src` to `target` func Snat4(target, src net.IP, chain, comment string) error { ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4) diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index c563c4e832..7b9516fe79 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -201,7 +201,7 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap // Note: the maximum length for linux interface name is 15 hostVethName = generateHostVethName(conf.VethPrefix, string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_NAME)) - err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCV4Cidrs, r.UseExternalSNAT, mtu, log) + err = driverClient.SetupNS(hostVethName, args.IfName, args.Netns, v4Addr, v6Addr, int(r.DeviceNumber), r.VPCv4CIDRs, r.UseExternalSNAT, mtu, log) } if err != nil { @@ -232,8 +232,8 @@ func add(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap containerInterfaceIndex := 1 ips := []*current.IPConfig{ { - Version: addrFamily, - Address: *addr, + Version: addrFamily, + Address: *addr, Interface: &containerInterfaceIndex, }, } diff --git a/cmd/routed-eni-cni-plugin/driver/driver.go b/cmd/routed-eni-cni-plugin/driver/driver.go index 63d0fed93e..b811700619 100644 --- a/cmd/routed-eni-cni-plugin/driver/driver.go +++ b/cmd/routed-eni-cni-plugin/driver/driver.go @@ -146,13 +146,6 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container's lo interface") } } - - //Enable v6 forwarding on Container's veth interface. We will set the same on Host side veth after we move it to host netns. - if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/forwarding", createVethContext.contVethName), "1"); err != nil { - if !os.IsNotExist(err) { - return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 forwarding container veth interface") - } - } } // Add a connected route to a dummy next hop (169.254.1.1 or fe80::1) @@ -160,24 +153,23 @@ func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error { // default via 169.254.1.1 dev eth0 // 169.254.1.1 dev eth0 - var gwIP string + var gw net.IP var maskLen int var addr *netlink.Addr var defNet *net.IPNet if createVethContext.v4Addr != nil { - gwIP = "169.254.1.1" + gw = net.IPv4(169, 254, 1, 1) maskLen = 32 addr = &netlink.Addr{IPNet: createVethContext.v4Addr} - _, defNet, _ = net.ParseCIDR("0.0.0.0/0") + defNet = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, maskLen)} } else if createVethContext.v6Addr != nil { - gwIP = "fe80::1" + gw = net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} maskLen = 128 addr = &netlink.Addr{IPNet: createVethContext.v6Addr} - _, defNet, _ = net.ParseCIDR("::/0") + defNet = &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, maskLen)} } - gw := net.ParseIP(gwIP) gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(maskLen, maskLen)} if err = createVethContext.netLink.RouteReplace(&netlink.Route{ diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index c4dcf32096..daaebe47fd 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -211,9 +211,9 @@ type EC2InstanceMetadataCache struct { availabilityZone string region string - unmanagedENIs StringSet - useCustomNetworking bool - cniunmanagedENIs StringSet + unmanagedENIs StringSet + useCustomNetworking bool + cniunmanagedENIs StringSet enablePrefixDelegation bool clusterName string @@ -623,7 +623,7 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat // If IPv6 is enabled, get attached v6 prefixes. if cache.v6Enabled { - imdsIPv6Prefixes, err := cache.imds.GetLocalIPv6Prefixes(ctx, eniMAC) + imdsIPv6Prefixes, err := cache.imds.GetIPv6Prefixes(ctx, eniMAC) if err != nil { return ENIMetadata{}, err } @@ -637,7 +637,7 @@ func (cache *EC2InstanceMetadataCache) getENIMetadata(eniMAC string) (ENIMetadat // If primary ENI has prefixes attached and then we move to custom networking, we don't need to fetch // the prefix since recommendation is to terminate the nodes and that would have deleted the prefix on the // primary ENI. - imdsIPv4Prefixes, err := cache.imds.GetLocalIPv4Prefixes(ctx, eniMAC) + imdsIPv4Prefixes, err := cache.imds.GetIPv4Prefixes(ctx, eniMAC) if err != nil { return ENIMetadata{}, err } @@ -1034,8 +1034,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv4sFromEC2(eniID string) (addrList [ // GetIPv4PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv4PrefixSpecification, err error) { - eniIds := make([]*string, 0) - eniIds = append(eniIds, aws.String(eniID)) + eniIds := []*string{aws.String(eniID)} input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} start := time.Now() @@ -1063,8 +1062,7 @@ func (cache *EC2InstanceMetadataCache) GetIPv4PrefixesFromEC2(eniID string) (add // GetIPv6PrefixesFromEC2 calls EC2 and returns a list of all addresses on the ENI func (cache *EC2InstanceMetadataCache) GetIPv6PrefixesFromEC2(eniID string) (addrList []*ec2.Ipv6PrefixSpecification, err error) { - eniIds := make([]*string, 0) - eniIds = append(eniIds, aws.String(eniID)) + eniIds := []*string{aws.String(eniID)} input := &ec2.DescribeNetworkInterfacesInput{NetworkInterfaceIds: eniIds} start := time.Now() diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index b7cd0265a6..d6f056761f 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -713,7 +713,6 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { PrivateIpAddress: &secondaryIP2, }, }, - //IPv4Prefixes: make([]*ec2.Ipv4PrefixSpecification, 0), IPv4Prefixes: nil, } eniList := []ENIMetadata{eni1Metadata, eni2Metadata} @@ -832,11 +831,11 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { } mockMetadata := testMetadata(map[string]interface{}{ metadataMACPath: primaryMAC + " " + eni2MAC, - metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, - metadataMACPath + eni2MAC + metadataInterface: eni2ID, - metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, - metadataMACPath + eni2MAC + metadataIPv4s: eniIPs, - metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes, + metadataMACPath + eni2MAC + metadataDeviceNum: eni2Device, + metadataMACPath + eni2MAC + metadataInterface: eni2ID, + metadataMACPath + eni2MAC + metadataSubnetCIDR: subnetCIDR, + metadataMACPath + eni2MAC + metadataIPv4s: eniIPs, + metadataMACPath + eni2MAC + metaDataPrefixPath: eniPrefixes, }) cache := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}, ec2SVC: mockEC2, enablePrefixDelegation: true, v4Enabled: tt.args.v4Enabled, v6Enabled: tt.args.v6Enabled} @@ -852,7 +851,6 @@ func TestEC2InstanceMetadataCache_waitForENIAndPrefixesAttached(t *testing.T) { } } - func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { mockMetadata := testMetadata(nil) ins := &EC2InstanceMetadataCache{imds: TypedIMDS{mockMetadata}} diff --git a/pkg/awsutils/imds.go b/pkg/awsutils/imds.go index a26580e678..4c0141b2de 100644 --- a/pkg/awsutils/imds.go +++ b/pkg/awsutils/imds.go @@ -93,11 +93,6 @@ func (imds TypedIMDS) GetLocalIPv4(ctx context.Context) (net.IP, error) { return imds.getIP(ctx, "local-ipv4") } -// GetLocalIPv6s returns the IPv6 addresses of the instance. -func (imds TypedIMDS) GetLocalIPv6s(ctx context.Context) (net.IP, error) { - return imds.getIP(ctx, "ipv6s") -} - // GetInstanceID returns the ID of this instance. func (imds TypedIMDS) GetInstanceID(ctx context.Context) (string, error) { instanceID, err := imds.GetMetadataWithContext(ctx, "instance-id") @@ -292,8 +287,8 @@ func (imds TypedIMDS) GetLocalIPv4s(ctx context.Context, mac string) ([]net.IP, return ips, err } -// GetLocalIPv4Prefixes returns the IPv4 prefixes delegated to this interface -func (imds TypedIMDS) GetLocalIPv4Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { +// GetIPv4Prefixes returns the IPv4 prefixes delegated to this interface +func (imds TypedIMDS) GetIPv4Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { key := fmt.Sprintf("network/interfaces/macs/%s/ipv4-prefix", mac) prefixes, err := imds.getCIDRs(ctx, key) if err != nil { @@ -309,8 +304,8 @@ func (imds TypedIMDS) GetLocalIPv4Prefixes(ctx context.Context, mac string) ([]n return prefixes, err } -// GetLocalIPv6Prefixes returns the IPv6 prefixes delegated to this interface -func (imds TypedIMDS) GetLocalIPv6Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { +// GetIPv6Prefixes returns the IPv6 prefixes delegated to this interface +func (imds TypedIMDS) GetIPv6Prefixes(ctx context.Context, mac string) ([]net.IPNet, error) { key := fmt.Sprintf("network/interfaces/macs/%s/ipv6-prefix", mac) prefixes, err := imds.getCIDRs(ctx, key) if err != nil { @@ -366,7 +361,7 @@ func (imds TypedIMDS) GetVPCIPv4CIDRBlocks(ctx context.Context, mac string) ([]n // GetVPCIPv6CIDRBlocks returns the IPv6 CIDR blocks for the VPC. func (imds TypedIMDS) GetVPCIPv6CIDRBlocks(ctx context.Context, mac string) ([]net.IPNet, error) { - key := fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-blocks", mac) + key := fmt.Sprintf("network/interfaces/macs/%s/vpc-ipv6-cidr-blocks", mac) ipnets, err := imds.getCIDRs(ctx, key) if err != nil { if imdsErr, ok := err.(*imdsRequestError); ok { @@ -382,6 +377,12 @@ func (imds TypedIMDS) GetVPCIPv6CIDRBlocks(ctx context.Context, mac string) ([]n return ipnets, err } +// GetSubnetIPv6CIDRBlocks returns the IPv4 CIDR block for the subnet in which the interface resides. +func (imds TypedIMDS) GetSubnetIPv6CIDRBlocks(ctx context.Context, mac string) (net.IPNet, error) { + key := fmt.Sprintf("network/interfaces/macs/%s/subnet-ipv6-cidr-blocks", mac) + return imds.getCIDR(ctx, key) +} + // IsNotFound returns true if the error was caused by an AWS API 404 response. func IsNotFound(err error) bool { if err != nil { diff --git a/pkg/awsutils/imds_test.go b/pkg/awsutils/imds_test.go index 409e61aebb..07ce476893 100644 --- a/pkg/awsutils/imds_test.go +++ b/pkg/awsutils/imds_test.go @@ -212,9 +212,22 @@ func TestGetVPCIPv4CIDRBlocks(t *testing.T) { } } +func TestGetSubnetIPv6CIDRBlocks(t *testing.T) { + f := TypedIMDS{FakeIMDS(map[string]interface{}{ + "network/interfaces/macs/02:c5:f8:3e:6b:27/subnet-ipv6-cidr-blocks": "2001:db8::/56", + })} + + ips, err := f.GetSubnetIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") + if assert.NoError(t, err) { + assert.Equal(t, ips, + net.IPNet{IP: net.IP{0x20, 0x1, 0xd, 0xb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}) + } +} + func TestGetVPCIPv6CIDRBlocks(t *testing.T) { f := TypedIMDS{FakeIMDS(map[string]interface{}{ - "network/interfaces/macs/02:c5:f8:3e:6b:27/subnet-ipv6-cidr-blocks": "2001:db8::/64", + "network/interfaces/macs/02:c5:f8:3e:6b:27/vpc-ipv6-cidr-blocks": "2001:db8::/64", })} ips, err := f.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") @@ -223,21 +236,32 @@ func TestGetVPCIPv6CIDRBlocks(t *testing.T) { []net.IPNet{{IP: net.IP{0x20, 0x1, 0xd, 0xb8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, Mask: net.IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}}) } + + nov6 := TypedIMDS{FakeIMDS(map[string]interface{}{ + // NB: IMDS returns 404, not empty string :( + })} + + ips, err = nov6.GetVPCIPv6CIDRBlocks(context.TODO(), "02:c5:f8:3e:6b:27") + if imdsErr, ok := err.(*imdsRequestError); ok { + if assert.NoError(t, imdsErr.err) { + assert.ElementsMatch(t, ips, []net.IP{}) + } + } } -func TestGetLocalIPv4Prefixes(t *testing.T) { +func TestGetIPv4Prefixes(t *testing.T) { f := TypedIMDS{FakeIMDS(map[string]interface{}{ "network/interfaces/macs/02:c5:f8:3e:6b:27/ipv4-prefix": `10.1.1.0/28`, })} - ips, err := f.GetLocalIPv4Prefixes(context.TODO(), "02:c5:f8:3e:6b:27") + ips, err := f.GetIPv4Prefixes(context.TODO(), "02:c5:f8:3e:6b:27") if imdsErr, ok := err.(*imdsRequestError); ok { if assert.NoError(t, imdsErr.err) { assert.Equal(t, ips, []net.IPNet{{IP: net.IPv4(10, 1, 1, 0), Mask: net.CIDRMask(28, 32)}}) } } - ips, err = f.GetLocalIPv4Prefixes(context.TODO(), "00:00:de:ad:be:ef") + ips, err = f.GetIPv4Prefixes(context.TODO(), "00:00:de:ad:be:ef") if imdsErr, ok := err.(*imdsRequestError); ok { if assert.NoError(t, imdsErr.err) { assert.ElementsMatch(t, ips, []net.IPNet{}) diff --git a/pkg/ec2wrapper/client.go b/pkg/ec2wrapper/client.go index e88ea29c63..f2b687c664 100644 --- a/pkg/ec2wrapper/client.go +++ b/pkg/ec2wrapper/client.go @@ -30,8 +30,8 @@ type EC2 interface { DetachNetworkInterfaceWithContext(ctx aws.Context, input *ec2svc.DetachNetworkInterfaceInput, opts ...request.Option) (*ec2svc.DetachNetworkInterfaceOutput, error) AssignPrivateIpAddressesWithContext(ctx aws.Context, input *ec2svc.AssignPrivateIpAddressesInput, opts ...request.Option) (*ec2svc.AssignPrivateIpAddressesOutput, error) UnassignPrivateIpAddressesWithContext(ctx aws.Context, input *ec2svc.UnassignPrivateIpAddressesInput, opts ...request.Option) (*ec2svc.UnassignPrivateIpAddressesOutput, error) - AssignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.AssignIpv6AddressesInput, opts ...request.Option) (*ec2svc.AssignIpv6AddressesOutput, error) - UnassignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.UnassignIpv6AddressesInput, opts ...request.Option) (*ec2svc.UnassignIpv6AddressesOutput, error) + AssignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.AssignIpv6AddressesInput, opts ...request.Option) (*ec2svc.AssignIpv6AddressesOutput, error) + UnassignIpv6AddressesWithContext(ctx aws.Context, input *ec2svc.UnassignIpv6AddressesInput, opts ...request.Option) (*ec2svc.UnassignIpv6AddressesOutput, error) DescribeNetworkInterfacesWithContext(ctx aws.Context, input *ec2svc.DescribeNetworkInterfacesInput, opts ...request.Option) (*ec2svc.DescribeNetworkInterfacesOutput, error) ModifyNetworkInterfaceAttributeWithContext(ctx aws.Context, input *ec2svc.ModifyNetworkInterfaceAttributeInput, opts ...request.Option) (*ec2svc.ModifyNetworkInterfaceAttributeOutput, error) CreateTagsWithContext(ctx aws.Context, input *ec2svc.CreateTagsInput, opts ...request.Option) (*ec2svc.CreateTagsOutput, error) diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 2c10f3a288..cb325e4104 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -78,8 +78,6 @@ const checkpointMigrationPhase = 1 const backfillNetworkName = "_migrated-from-cri" const backfillNetworkIface = "unknown" -const defaultMaxIPv6addresses = 250 - // ErrUnknownPod is an error when there is no pod in data store matching pod name, namespace, sandbox id var ErrUnknownPod = errors.New("datastore: unknown pod") @@ -344,7 +342,8 @@ type CheckpointData struct { // in checkpoints. type CheckpointEntry struct { IPAMKey - IP string `json:"ip"` + IPv4 string `json:"ipv4,omitempty"` + IPv6 string `json:"ipv6,omitempty"` } // ReadBackingStore initialises the IP allocation state from the @@ -357,7 +356,7 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { case 1: // Phase1: Read from CRI ds.log.Infof("Reading ipam state from CRI") - + var ipv4Addr, ipv6Addr string sandboxes, err := ds.cri.GetRunningPodSandboxes(ds.log) if err != nil { return err @@ -366,6 +365,11 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { entries := make([]CheckpointEntry, 0, len(sandboxes)) for _, s := range sandboxes { ds.log.Debugf("Adding container ID: %v", s.ID) + if isv6Enabled { + ipv6Addr = s.IP + } else { + ipv4Addr = s.IP + } entries = append(entries, CheckpointEntry{ // NB: These Backfill values are also assumed in UnassignPodIPAddress IPAMKey: IPAMKey{ @@ -373,7 +377,8 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { ContainerID: s.ID, IfName: backfillNetworkIface, }, - IP: s.IP, + IPv4: ipv4Addr, + IPv6: ipv6Addr, }) } data = CheckpointData{ @@ -409,25 +414,29 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { defer ds.lock.Unlock() for _, allocation := range data.Allocations { - ipAddr := net.ParseIP(allocation.IP) + ipv4Addr := net.ParseIP(allocation.IPv4) + ipv6Addr := net.ParseIP(allocation.IPv6) + var ipAddr net.IP found := false eniloop: for _, eni := range ds.eniPool { eniCidrs := eni.AvailableIPv4Cidrs + ipAddr = ipv4Addr if isv6Enabled { ds.log.Debugf("v6 is enabled") eniCidrs = eni.IPv6Cidrs + ipAddr = ipv6Addr } for _, cidr := range eniCidrs { ds.log.Debugf("Checking if IP: %v belongs to CIDR: %v", ipAddr, cidr.Cidr) if cidr.Cidr.Contains(ipAddr) { // Found! found = true - if _, ok := cidr.IPAddresses[allocation.IP]; ok { + if _, ok := cidr.IPAddresses[ipAddr.String()]; ok { return errors.New(IPAlreadyInStoreError) } addr := &AddressInfo{Address: ipAddr.String()} - cidr.IPAddresses[allocation.IP] = addr + cidr.IPAddresses[ipAddr.String()] = addr ds.assignPodIPAddressUnsafe(allocation.IPAMKey, eni, addr) ds.log.Debugf("Recovered %s => %s/%s", allocation.IPAMKey, eni.ID, addr.Address) //Update prometheus for ips per cidr @@ -438,7 +447,8 @@ func (ds *DataStore) ReadBackingStore(isv6Enabled bool) error { } } if !found { - ds.log.Infof("datastore: Sandbox %s uses unknown IP Address %s - presuming stale/dead", allocation.IPAMKey, allocation.IP) + ds.log.Infof("datastore: Sandbox %s uses unknown IP Address %s - presuming stale/dead", + allocation.IPAMKey, ipAddr.String()) } } @@ -466,7 +476,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { if addr.Assigned() { entry := CheckpointEntry{ IPAMKey: addr.IPAMKey, - IP: addr.Address, + IPv4: addr.Address, } allocations = append(allocations, entry) } @@ -478,7 +488,7 @@ func (ds *DataStore) writeBackingStoreUnsafe() error { if addr.Assigned() { entry := CheckpointEntry{ IPAMKey: addr.IPAMKey, - IP: addr.Address, + IPv6: addr.Address, } allocations = append(allocations, entry) } @@ -539,7 +549,7 @@ func (ds *DataStore) AddIPv4CidrToStore(eniID string, ipv4Cidr net.IPNet, isPref newCidrInfo := &CidrInfo{ Cidr: ipv4Cidr, - IPAddresses: make(map[string]*AddressInfo), + IPAddresses: make(map[string]*AddressInfo), IsPrefix: isPrefix, AddressFamily: "4", } @@ -630,18 +640,16 @@ func (ds *DataStore) AddIPv6CidrToStore(eniID string, ipv6Cidr net.IPNet, isPref } ds.log.Debugf("Assigning IPv6CIDRs") - if curENI.IPv6Cidrs == nil { curENI.IPv6Cidrs = make(map[string]*CidrInfo) } + if curENI.IPv6Cidrs == nil { + curENI.IPv6Cidrs = make(map[string]*CidrInfo) + } curENI.IPv6Cidrs[strIPv6Cidr] = &CidrInfo{ Cidr: ipv6Cidr, IPAddresses: make(map[string]*AddressInfo), IsPrefix: isPrefix, AddressFamily: "6", } - - //curENI.IPv6Cidrs[strIPv6Cidr].Size() will end up with a huge number. So, instead capping it to Max Pods ceiling on - //an EKS Node. TODO - Should we increase the default value further as VPC CNI can also be used in a Self Managed K8S - //cluster with no such upper bound? - ds.total += defaultMaxIPv6addresses + ds.total += curENI.IPv6Cidrs[strIPv6Cidr].Size() if isPrefix { ds.allocatedPrefix++ } @@ -830,7 +838,7 @@ func (ds *DataStore) GetStats(addressFamily string) (int, int, int) { assignedIPs += cidr.AssignedIPAddressesInCidr() //Set to default Max pods we support on an instance, otherwise we will end up displaying //a huge number. - totalIPs += defaultMaxIPv6addresses + totalIPs += cidr.Size() } } } @@ -1233,9 +1241,9 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { tmpENIInfo.IPv6Cidrs = make(map[string]*CidrInfo, len(eniInfo.IPv6Cidrs)) for cidr, _ := range eniInfo.AvailableIPv4Cidrs { tmpENIInfo.AvailableIPv4Cidrs[cidr] = &CidrInfo{ - Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr, + Cidr: eniInfo.AvailableIPv4Cidrs[cidr].Cidr, IPAddresses: make(map[string]*AddressInfo, len(eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses)), - IsPrefix: eniInfo.AvailableIPv4Cidrs[cidr].IsPrefix, + IsPrefix: eniInfo.AvailableIPv4Cidrs[cidr].IsPrefix, } // Since IP Addresses might get removed, we need to make a deep copy here. for ip, ipAddrInfoRef := range eniInfo.AvailableIPv4Cidrs[cidr].IPAddresses { @@ -1245,9 +1253,9 @@ func (ds *DataStore) GetENIInfos() *ENIInfos { } for cidr, _ := range eniInfo.IPv6Cidrs { tmpENIInfo.IPv6Cidrs[cidr] = &CidrInfo{ - Cidr: eniInfo.IPv6Cidrs[cidr].Cidr, + Cidr: eniInfo.IPv6Cidrs[cidr].Cidr, IPAddresses: make(map[string]*AddressInfo, len(eniInfo.IPv6Cidrs[cidr].IPAddresses)), - IsPrefix: eniInfo.IPv6Cidrs[cidr].IsPrefix, + IsPrefix: eniInfo.IPv6Cidrs[cidr].IsPrefix, } // Since IP Addresses might get removed, we need to make a deep copy here. for ip, ipAddrInfoRef := range eniInfo.IPv6Cidrs[cidr].IPAddresses { @@ -1412,7 +1420,7 @@ func (ds *DataStore) FindFreeableCidrs(eniID string) []CidrInfo { if assignedaddr.AssignedIPAddressesInCidr() == 0 { tempFreeable := CidrInfo{ Cidr: assignedaddr.Cidr, - IPAddresses: nil, + IPAddresses: nil, IsPrefix: assignedaddr.IsPrefix, AddressFamily: assignedaddr.AddressFamily, } diff --git a/pkg/ipamd/datastore/data_store_test.go b/pkg/ipamd/datastore/data_store_test.go index 07f959514b..0e8d03403a 100644 --- a/pkg/ipamd/datastore/data_store_test.go +++ b/pkg/ipamd/datastore/data_store_test.go @@ -255,7 +255,7 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, checkpoint.Data, &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IP: "1.1.1.1"}, + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, }, }) @@ -285,7 +285,7 @@ func TestPodIPv4Address(t *testing.T) { assert.Equal(t, checkpoint.Data, &CheckpointData{ Version: CheckpointFormatVersion, Allocations: []CheckpointEntry{ - {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IP: "1.1.1.1"}, + {IPAMKey: IPAMKey{NetworkName: "net0", ContainerID: "sandbox-1", IfName: "eth0"}, IPv4: "1.1.1.1"}, }, }) checkpoint.Error = nil diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index a6c8b583d0..d507a26a83 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -146,7 +146,6 @@ const ( ipV6AddrFamily = "6" ) - var log = logger.Get() var ( @@ -443,7 +442,6 @@ func (c *IPAMContext) nodeInit() error { return err } - if c.enableIPv6 { //We will not support upgrading/converting an existing IPv4 cluster to operate in IPv6 mode. So, we will always //start with a clean slate in IPv6 mode. We also don't have to deal with dynamic update of Prefix Delegation @@ -871,8 +869,7 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) { //Let's make an EC2 API call to get a list of IPv6 prefixes (if any) that are already attached to the //current ENI. We will make this call only once during boot up/init and doing so will shield us from any //IMDS out of sync issues. We only need one v6 prefix per ENI/Node. - var ec2v6Prefixes []*ec2.Ipv6PrefixSpecification - ec2v6Prefixes, err = c.awsClient.GetIPv6PrefixesFromEC2(eniID) + ec2v6Prefixes, err := c.awsClient.GetIPv6PrefixesFromEC2(eniID) if err != nil { log.Errorf("assignIPv6Prefix; err: %s", err) return err @@ -896,6 +893,11 @@ func (c *IPAMContext) assignIPv6Prefix(eniID string) (err error) { ec2v6Prefixes = append(ec2v6Prefixes, &ec2.Ipv6PrefixSpecification{Ipv6Prefix: v6Prefix}) } log.Debugf("Successfully allocated an IPv6Prefix for ENI: %s", eniID) + } else if len(ec2v6Prefixes) > 1 { + //Found more than one v6 prefix attached to the ENI. VPC CNI will only attach a single v6 prefix + //and it will not attempt to free any additional Prefixes that are already attached. + //Will use the first IPv6 Prefix attached for IP address allocation. + ec2v6Prefixes = []*ec2.Ipv6PrefixSpecification{ec2v6Prefixes[0]} } c.addENIv6prefixesToDataStore(ec2v6Prefixes, eniID) return nil @@ -2082,6 +2084,5 @@ func (c *IPAMContext) isConfigValid() bool { c.enablePrefixDelegation = false } - return true } diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index d109a5a4cf..6f7061568a 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -102,7 +102,7 @@ func TestNodeInit(t *testing.T) { fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddr02}, + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddr02}, }, } @@ -186,7 +186,7 @@ func TestNodeInitwithPDenabledIPv4Mode(t *testing.T) { fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddrPD01}, + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv4: ipaddrPD01}, }, } @@ -269,7 +269,7 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { fakeCheckpoint := datastore.CheckpointData{ Version: datastore.CheckpointFormatVersion, Allocations: []datastore.CheckpointEntry{ - {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IP: ipaddrPD01}, + {IPAMKey: datastore.IPAMKey{NetworkName: "net0", ContainerID: "sandbox-id", IfName: "eth0"}, IPv6: ipaddrPD01}, }, } @@ -307,7 +307,6 @@ func TestNodeInitwithPDenabledIPv6Mode(t *testing.T) { m.awsutils.EXPECT().GetPrimaryENImac().Return(eni1.MAC) m.awsutils.EXPECT().IsPrimaryENI(primaryENIid).Return(true).AnyTimes() - eniMetadataSlice := []awsutils.ENIMetadata{eni1} resp := awsutils.DescribeAllENIsResult{ ENIMetadata: eniMetadataSlice, diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index 0402904d7f..b00cffcf9d 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -140,7 +140,6 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp IfName: in.IfName, NetworkName: in.NetworkName, } - //addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPv4Address(ipamKey) ipv4Addr, ipv6Addr, deviceNumber, err = s.ipamContext.dataStore.AssignPodIPAddress(ipamKey, s.ipamContext.enableIPv4, s.ipamContext.enableIPv6) } @@ -177,8 +176,8 @@ func (s *server) AddNetwork(ctx context.Context, in *rpc.AddNetworkRequest) (*rp IPv6Addr: ipv6Addr, DeviceNumber: int32(deviceNumber), UseExternalSNAT: useExternalSNAT, - VPCV4Cidrs: pbVPCV4cidrs, - VPCV6Cidrs: pbVPCV6cidrs, + VPCv4CIDRs: pbVPCV4cidrs, + VPCv6CIDRs: pbVPCV6cidrs, PodVlanId: int32(vlanID), PodENIMAC: branchENIMAC, PodENISubnetGW: podENISubnetGW, diff --git a/pkg/ipamd/rpc_handler_test.go b/pkg/ipamd/rpc_handler_test.go index eae015ce3b..4ddb74c93f 100644 --- a/pkg/ipamd/rpc_handler_test.go +++ b/pkg/ipamd/rpc_handler_test.go @@ -137,7 +137,7 @@ func TestServer_AddNetwork(t *testing.T) { IPv4Addr: "192.168.1.100", DeviceNumber: int32(0), UseExternalSNAT: true, - VPCV4Cidrs: []string{"10.10.0.0/16"}, + VPCv4CIDRs: []string{"10.10.0.0/16"}, }, }, { @@ -169,7 +169,7 @@ func TestServer_AddNetwork(t *testing.T) { IPv4Addr: "192.168.1.100", DeviceNumber: int32(0), UseExternalSNAT: false, - VPCV4Cidrs: []string{"10.10.0.0/16", "10.12.0.0/16", "10.13.0.0/16"}, + VPCv4CIDRs: []string{"10.10.0.0/16", "10.12.0.0/16", "10.13.0.0/16"}, }, }, { @@ -203,7 +203,7 @@ func TestServer_AddNetwork(t *testing.T) { Success: true, IPv6Addr: "2001:db8::", DeviceNumber: int32(0), - VPCV6Cidrs: []string{"2001:db8::/56"}, + VPCv6CIDRs: []string{"2001:db8::/56"}, }, }, { diff --git a/pkg/networkutils/network.go b/pkg/networkutils/network.go index 9635a350b2..78286cea59 100644 --- a/pkg/networkutils/network.go +++ b/pkg/networkutils/network.go @@ -239,12 +239,35 @@ func (n *linuxNetwork) enableIPv6() (err error) { } } - //Let's disable ipv6 forwarding on eth0 (Primary ENI). We will direct external v6 traffic via GW - if err = n.procSys.Set(fmt.Sprintf("net/ipv6/conf/eth0/forwarding"), "0"); err != nil { + //Set accept_ra to '2' for eth0. Allows us to overrule forwarding behaviour for the Node's Primary interface and + //will allow eth0 to accept Router Advertisements even though forwarding is enabled on *all* interfaces on the host netns. + if err = n.procSys.Set(fmt.Sprintf("net/ipv6/conf/eth0/accept_ra"), "2"); err != nil { if !os.IsNotExist(err) { - return errors.Wrapf(err, "setupVeth network: failed to disable IPv6 forwarding on eth0 interface") + return errors.Wrapf(err, "setupVeth network: failed to set accept_ra to 2 for eth0") } } + + if err = n.setupRuleToBlockNodeLocalV4Access(); err != nil { + return errors.Wrapf(err, "setupVeth network: failed to setup route to block pod access via IPv4 address") + } + return nil +} + +func (n *linuxNetwork) SetupRuleToBlockNodeLocalV4Access() error { + return n.setupRuleToBlockNodeLocalV4Access() +} + +//Setup a rule to block traffic directed to v4 interface of the Pod +func (n *linuxNetwork) setupRuleToBlockNodeLocalV4Access() error { + ipt, err := n.newIptables(iptables.ProtocolIPv4) + if err != nil { + return errors.Wrap(err, "failed to create iptables") + } + + if err := ipt.Insert("filter", "FORWARD", 1, "-d", "169.254.172.0/22", "-m", "conntrack", + "--ctstate", "NEW", "-m", "comment", "--comment", "Block Node Local Pod access via IPv4", "-j", "REJECT"); err != nil { + return fmt.Errorf("failed adding v4 drop route: %v", err) + } return nil } @@ -366,7 +389,7 @@ func (n *linuxNetwork) updateHostIptablesRules(vpcCIDRs []string, primaryMAC str } ipProtocol := iptables.ProtocolIPv4 - if v6Enabled{ + if v6Enabled { //Essentially a stub function for now in V6 mode. We will need it when we support v6 in secondary IP and //custom networking modes. We don't need to install any SNAT rules in v6 mode and currently there is no need //to mark packets entering via Primary ENI as all the pods in v6 mode will be behind primary ENI. Will have to diff --git a/pkg/networkutils/network_test.go b/pkg/networkutils/network_test.go index c174d10f4a..935dd7bc57 100644 --- a/pkg/networkutils/network_test.go +++ b/pkg/networkutils/network_test.go @@ -752,12 +752,24 @@ func TestSetupHostNetworkWithIPv6Enabled(t *testing.T) { setupNetLinkMocks(ctrl, mockNetLink) mockProcSys.EXPECT().Set("net/ipv6/conf/all/disable_ipv6", "0").Return(nil) mockProcSys.EXPECT().Set("net/ipv6/conf/all/forwarding", "1").Return(nil) - mockProcSys.EXPECT().Set("net/ipv6/conf/eth0/forwarding", "0").Return(nil) - + mockProcSys.EXPECT().Set("net/ipv6/conf/eth0/accept_ra", "2").Return(nil) var vpcCIDRs []string err := ln.SetupHostNetwork(vpcCIDRs, loopback, &testENINetIP, false, false, true) assert.NoError(t, err) + + assert.Equal(t, map[string]map[string][][]string{ + "filter": { + "FORWARD": [][]string{ + { + "-d", "169.254.172.0/22", + "-m", "conntrack", "--ctstate", "NEW", + "-m", "comment", "--comment", "Block Node Local Pod access via IPv4", + "-j", "REJECT", + }, + }, + }, + }, mockIptables.dataplaneState) } func TestIncrementIPv4Addr(t *testing.T) { @@ -878,6 +890,10 @@ func (ipt *mockIptables) Exists(table, chainName string, rulespec ...string) (bo } func (ipt *mockIptables) Insert(table, chain string, pos int, rulespec ...string) error { + if ipt.dataplaneState[table] == nil { + ipt.dataplaneState[table] = map[string][][]string{} + } + ipt.dataplaneState[table][chain] = append(ipt.dataplaneState[table][chain], rulespec) return nil } diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 33974d7670..02ac894713 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -2,7 +2,7 @@ // versions: // protoc-gen-go v1.26.0 // protoc v3.17.2 -// source: rpc/rpc.proto +// source: rpc.proto package rpc @@ -38,7 +38,7 @@ type AddNetworkRequest struct { func (x *AddNetworkRequest) Reset() { *x = AddNetworkRequest{} if protoimpl.UnsafeEnabled { - mi := &file_rpc_rpc_proto_msgTypes[0] + mi := &file_rpc_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -51,7 +51,7 @@ func (x *AddNetworkRequest) String() string { func (*AddNetworkRequest) ProtoMessage() {} func (x *AddNetworkRequest) ProtoReflect() protoreflect.Message { - mi := &file_rpc_rpc_proto_msgTypes[0] + mi := &file_rpc_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -64,7 +64,7 @@ func (x *AddNetworkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AddNetworkRequest.ProtoReflect.Descriptor instead. func (*AddNetworkRequest) Descriptor() ([]byte, []int) { - return file_rpc_rpc_proto_rawDescGZIP(), []int{0} + return file_rpc_proto_rawDescGZIP(), []int{0} } func (x *AddNetworkRequest) GetClientVersion() string { @@ -133,19 +133,19 @@ type AddNetworkReply struct { IPv6Addr string `protobuf:"bytes,3,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"` DeviceNumber int32 `protobuf:"varint,4,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"` UseExternalSNAT bool `protobuf:"varint,5,opt,name=UseExternalSNAT,proto3" json:"UseExternalSNAT,omitempty"` - VPCV4Cidrs []string `protobuf:"bytes,6,rep,name=VPCV4cidrs,proto3" json:"VPCV4cidrs,omitempty"` - VPCV6Cidrs []string `protobuf:"bytes,7,rep,name=VPCV6cidrs,proto3" json:"VPCV6cidrs,omitempty"` + VPCv4CIDRs []string `protobuf:"bytes,6,rep,name=VPCv4CIDRs,proto3" json:"VPCv4CIDRs,omitempty"` // start of pod-eni parameters - PodVlanId int32 `protobuf:"varint,8,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` - PodENIMAC string `protobuf:"bytes,9,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` - PodENISubnetGW string `protobuf:"bytes,10,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` - ParentIfIndex int32 `protobuf:"varint,11,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` // end of pod-eni parameters + PodVlanId int32 `protobuf:"varint,7,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` + PodENIMAC string `protobuf:"bytes,8,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` + PodENISubnetGW string `protobuf:"bytes,9,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` + ParentIfIndex int32 `protobuf:"varint,10,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` // end of pod-eni parameters + VPCv6CIDRs []string `protobuf:"bytes,11,rep,name=VPCv6CIDRs,proto3" json:"VPCv6CIDRs,omitempty"` } func (x *AddNetworkReply) Reset() { *x = AddNetworkReply{} if protoimpl.UnsafeEnabled { - mi := &file_rpc_rpc_proto_msgTypes[1] + mi := &file_rpc_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -158,7 +158,7 @@ func (x *AddNetworkReply) String() string { func (*AddNetworkReply) ProtoMessage() {} func (x *AddNetworkReply) ProtoReflect() protoreflect.Message { - mi := &file_rpc_rpc_proto_msgTypes[1] + mi := &file_rpc_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -171,7 +171,7 @@ func (x *AddNetworkReply) ProtoReflect() protoreflect.Message { // Deprecated: Use AddNetworkReply.ProtoReflect.Descriptor instead. func (*AddNetworkReply) Descriptor() ([]byte, []int) { - return file_rpc_rpc_proto_rawDescGZIP(), []int{1} + return file_rpc_proto_rawDescGZIP(), []int{1} } func (x *AddNetworkReply) GetSuccess() bool { @@ -209,16 +209,9 @@ func (x *AddNetworkReply) GetUseExternalSNAT() bool { return false } -func (x *AddNetworkReply) GetVPCV4Cidrs() []string { +func (x *AddNetworkReply) GetVPCv4CIDRs() []string { if x != nil { - return x.VPCV4Cidrs - } - return nil -} - -func (x *AddNetworkReply) GetVPCV6Cidrs() []string { - if x != nil { - return x.VPCV6Cidrs + return x.VPCv4CIDRs } return nil } @@ -251,6 +244,13 @@ func (x *AddNetworkReply) GetParentIfIndex() int32 { return 0 } +func (x *AddNetworkReply) GetVPCv6CIDRs() []string { + if x != nil { + return x.VPCv6CIDRs + } + return nil +} + type DelNetworkRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -269,7 +269,7 @@ type DelNetworkRequest struct { func (x *DelNetworkRequest) Reset() { *x = DelNetworkRequest{} if protoimpl.UnsafeEnabled { - mi := &file_rpc_rpc_proto_msgTypes[2] + mi := &file_rpc_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -282,7 +282,7 @@ func (x *DelNetworkRequest) String() string { func (*DelNetworkRequest) ProtoMessage() {} func (x *DelNetworkRequest) ProtoReflect() protoreflect.Message { - mi := &file_rpc_rpc_proto_msgTypes[2] + mi := &file_rpc_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -295,7 +295,7 @@ func (x *DelNetworkRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DelNetworkRequest.ProtoReflect.Descriptor instead. func (*DelNetworkRequest) Descriptor() ([]byte, []int) { - return file_rpc_rpc_proto_rawDescGZIP(), []int{2} + return file_rpc_proto_rawDescGZIP(), []int{2} } func (x *DelNetworkRequest) GetClientVersion() string { @@ -369,7 +369,7 @@ type DelNetworkReply struct { func (x *DelNetworkReply) Reset() { *x = DelNetworkReply{} if protoimpl.UnsafeEnabled { - mi := &file_rpc_rpc_proto_msgTypes[3] + mi := &file_rpc_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -382,7 +382,7 @@ func (x *DelNetworkReply) String() string { func (*DelNetworkReply) ProtoMessage() {} func (x *DelNetworkReply) ProtoReflect() protoreflect.Message { - mi := &file_rpc_rpc_proto_msgTypes[3] + mi := &file_rpc_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -395,7 +395,7 @@ func (x *DelNetworkReply) ProtoReflect() protoreflect.Message { // Deprecated: Use DelNetworkReply.ProtoReflect.Descriptor instead. func (*DelNetworkReply) Descriptor() ([]byte, []int) { - return file_rpc_rpc_proto_rawDescGZIP(), []int{3} + return file_rpc_proto_rawDescGZIP(), []int{3} } func (x *DelNetworkReply) GetSuccess() bool { @@ -426,114 +426,114 @@ func (x *DelNetworkReply) GetPodVlanId() int32 { return 0 } -var File_rpc_rpc_proto protoreflect.FileDescriptor - -var file_rpc_rpc_proto_rawDesc = []byte{ - 0x0a, 0x0d, 0x72, 0x70, 0x63, 0x2f, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x03, 0x72, 0x70, 0x63, 0x22, 0xb5, 0x02, 0x0a, 0x11, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, - 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, - 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, - 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x12, 0x3a, - 0x0a, 0x1a, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x5f, - 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x43, - 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, - 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, - 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0xfb, 0x02, 0x0a, - 0x0f, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, - 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, - 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, - 0x64, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, - 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, - 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, - 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x34, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x34, 0x63, 0x69, 0x64, 0x72, 0x73, - 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x36, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x07, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x56, 0x36, 0x63, 0x69, 0x64, 0x72, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1c, - 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x12, 0x26, 0x0a, 0x0e, - 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x47, 0x57, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, - 0x65, 0x74, 0x47, 0x57, 0x12, 0x24, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x66, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x50, 0x61, 0x72, - 0x65, 0x6e, 0x74, 0x49, 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0xb7, 0x02, 0x0a, 0x11, 0x44, - 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, - 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, - 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, - 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, - 0x50, 0x41, 0x43, 0x45, 0x12, 0x3a, 0x0a, 0x1a, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, - 0x49, 0x4e, 0x46, 0x52, 0x41, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, - 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, - 0x49, 0x4e, 0x46, 0x52, 0x41, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, - 0x12, 0x16, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x66, - 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, - 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, - 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, - 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, - 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, +var File_rpc_proto protoreflect.FileDescriptor + +var file_rpc_proto_rawDesc = []byte{ + 0x0a, 0x09, 0x72, 0x70, 0x63, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x03, 0x72, 0x70, 0x63, + 0x22, 0xb5, 0x02, 0x0a, 0x11, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, + 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, + 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, + 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, + 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x12, 0x3a, 0x0a, 0x1a, 0x4b, 0x38, + 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x5f, 0x43, 0x4f, 0x4e, 0x54, + 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x49, 0x4e, 0x46, 0x52, 0x41, 0x43, 0x4f, 0x4e, 0x54, 0x41, + 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x49, 0x44, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x66, 0x4e, 0x61, + 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, + 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x4e, 0x65, 0x74, 0x6e, 0x73, 0x22, 0xfb, 0x02, 0x0a, 0x0f, 0x41, 0x64, 0x64, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, + 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, + 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, + 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, + 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, + 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, - 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, - 0x32, 0x88, 0x01, 0x0a, 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, - 0x3c, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, - 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, - 0x0a, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, - 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x72, - 0x70, 0x63, 0x2f, 0x3b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x72, 0x12, 0x28, 0x0a, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x55, 0x73, 0x65, + 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x12, 0x1e, 0x0a, 0x0a, + 0x56, 0x50, 0x43, 0x76, 0x34, 0x43, 0x49, 0x44, 0x52, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x34, 0x43, 0x49, 0x44, 0x52, 0x73, 0x12, 0x1c, 0x0a, 0x09, + 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, + 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, + 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x12, 0x26, 0x0a, 0x0e, 0x50, 0x6f, 0x64, 0x45, + 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x47, 0x57, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0e, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x47, 0x57, + 0x12, 0x24, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x66, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, + 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x36, 0x43, + 0x49, 0x44, 0x52, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, + 0x36, 0x43, 0x49, 0x44, 0x52, 0x73, 0x22, 0xb7, 0x02, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x4e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, + 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x4e, 0x41, + 0x4d, 0x45, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, + 0x4e, 0x41, 0x4d, 0x45, 0x12, 0x2a, 0x0a, 0x11, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, + 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0f, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x4e, 0x41, 0x4d, 0x45, 0x53, 0x50, 0x41, 0x43, 0x45, + 0x12, 0x3a, 0x0a, 0x1a, 0x4b, 0x38, 0x53, 0x5f, 0x50, 0x4f, 0x44, 0x5f, 0x49, 0x4e, 0x46, 0x52, + 0x41, 0x5f, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x5f, 0x49, 0x44, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x4b, 0x38, 0x53, 0x50, 0x4f, 0x44, 0x49, 0x4e, 0x46, 0x52, + 0x41, 0x43, 0x4f, 0x4e, 0x54, 0x41, 0x49, 0x4e, 0x45, 0x52, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, + 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x52, 0x65, + 0x61, 0x73, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x49, 0x44, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x43, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x49, 0x44, 0x12, 0x16, 0x0a, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, + 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, + 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, + 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, + 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, + 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, + 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x32, 0x88, 0x01, 0x0a, + 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3c, 0x0a, 0x0a, 0x41, + 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, + 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0a, 0x44, 0x65, 0x6c, + 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, + 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x72, 0x70, 0x63, 0x2f, 0x3b, + 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( - file_rpc_rpc_proto_rawDescOnce sync.Once - file_rpc_rpc_proto_rawDescData = file_rpc_rpc_proto_rawDesc + file_rpc_proto_rawDescOnce sync.Once + file_rpc_proto_rawDescData = file_rpc_proto_rawDesc ) -func file_rpc_rpc_proto_rawDescGZIP() []byte { - file_rpc_rpc_proto_rawDescOnce.Do(func() { - file_rpc_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_rpc_rpc_proto_rawDescData) +func file_rpc_proto_rawDescGZIP() []byte { + file_rpc_proto_rawDescOnce.Do(func() { + file_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(file_rpc_proto_rawDescData) }) - return file_rpc_rpc_proto_rawDescData + return file_rpc_proto_rawDescData } -var file_rpc_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 4) -var file_rpc_rpc_proto_goTypes = []interface{}{ +var file_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_rpc_proto_goTypes = []interface{}{ (*AddNetworkRequest)(nil), // 0: rpc.AddNetworkRequest (*AddNetworkReply)(nil), // 1: rpc.AddNetworkReply (*DelNetworkRequest)(nil), // 2: rpc.DelNetworkRequest (*DelNetworkReply)(nil), // 3: rpc.DelNetworkReply } -var file_rpc_rpc_proto_depIdxs = []int32{ +var file_rpc_proto_depIdxs = []int32{ 0, // 0: rpc.CNIBackend.AddNetwork:input_type -> rpc.AddNetworkRequest 2, // 1: rpc.CNIBackend.DelNetwork:input_type -> rpc.DelNetworkRequest 1, // 2: rpc.CNIBackend.AddNetwork:output_type -> rpc.AddNetworkReply @@ -545,13 +545,13 @@ var file_rpc_rpc_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for field type_name } -func init() { file_rpc_rpc_proto_init() } -func file_rpc_rpc_proto_init() { - if File_rpc_rpc_proto != nil { +func init() { file_rpc_proto_init() } +func file_rpc_proto_init() { + if File_rpc_proto != nil { return } if !protoimpl.UnsafeEnabled { - file_rpc_rpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_rpc_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AddNetworkRequest); i { case 0: return &v.state @@ -563,7 +563,7 @@ func file_rpc_rpc_proto_init() { return nil } } - file_rpc_rpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_rpc_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*AddNetworkReply); i { case 0: return &v.state @@ -575,7 +575,7 @@ func file_rpc_rpc_proto_init() { return nil } } - file_rpc_rpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_rpc_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DelNetworkRequest); i { case 0: return &v.state @@ -587,7 +587,7 @@ func file_rpc_rpc_proto_init() { return nil } } - file_rpc_rpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_rpc_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DelNetworkReply); i { case 0: return &v.state @@ -604,18 +604,18 @@ func file_rpc_rpc_proto_init() { out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_rpc_rpc_proto_rawDesc, + RawDescriptor: file_rpc_proto_rawDesc, NumEnums: 0, NumMessages: 4, NumExtensions: 0, NumServices: 1, }, - GoTypes: file_rpc_rpc_proto_goTypes, - DependencyIndexes: file_rpc_rpc_proto_depIdxs, - MessageInfos: file_rpc_rpc_proto_msgTypes, + GoTypes: file_rpc_proto_goTypes, + DependencyIndexes: file_rpc_proto_depIdxs, + MessageInfos: file_rpc_proto_msgTypes, }.Build() - File_rpc_rpc_proto = out.File - file_rpc_rpc_proto_rawDesc = nil - file_rpc_rpc_proto_goTypes = nil - file_rpc_rpc_proto_depIdxs = nil + File_rpc_proto = out.File + file_rpc_proto_rawDesc = nil + file_rpc_proto_goTypes = nil + file_rpc_proto_depIdxs = nil } diff --git a/rpc/rpc.proto b/rpc/rpc.proto index 930bed18e3..9fc2ca33b9 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -28,17 +28,18 @@ message AddNetworkReply { string IPv6Addr = 3; int32 DeviceNumber = 4; bool UseExternalSNAT = 5; - repeated string VPCV4cidrs = 6; - repeated string VPCV6cidrs = 7; + repeated string VPCv4CIDRs = 6; // start of pod-eni parameters - int32 PodVlanId = 8; - string PodENIMAC = 9; - string PodENISubnetGW = 10; - int32 ParentIfIndex = 11; + int32 PodVlanId = 7; + string PodENIMAC = 8; + string PodENISubnetGW = 9; + int32 ParentIfIndex = 10; // end of pod-eni parameters + repeated string VPCv6CIDRs = 11; + // next field: 12 } From be7e13fa1aaaa4b02b1e8337018ba6447d4f72fe Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Wed, 8 Sep 2021 11:38:52 -0700 Subject: [PATCH 3/7] Rearrange rpc proto fields --- cmd/routed-eni-cni-plugin/cni.go | 14 ++- pkg/ipamd/rpc_handler.go | 13 +- rpc/mocks/rpc_mocks.go | 3 +- rpc/rpc.pb.go | 207 +++++++++++++++++++++++++------ rpc/rpc.proto | 13 +- rpc/rpc_grpc.pb.go | 136 -------------------- 6 files changed, 201 insertions(+), 185 deletions(-) delete mode 100644 rpc/rpc_grpc.pb.go diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 7b9516fe79..562fef637a 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -328,11 +328,21 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap log.Infof("Received del network response for pod %s namespace %s sandbox %s: %+v", string(k8sArgs.K8S_POD_NAME), string(k8sArgs.K8S_POD_NAMESPACE), string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID), r) - deletedPodIP := net.ParseIP(r.IPv4Addr) + var deletedPodIP net.IP + var maskLen int + if r.IPv4Addr != "" { + deletedPodIP = net.ParseIP(r.IPv4Addr) + maskLen = 32 + } else if r.IPv6Addr != "" { + log.Debug("Deleting Pod NS - V6 Mode") + deletedPodIP = net.ParseIP(r.IPv6Addr) + maskLen = 128 + } + if deletedPodIP != nil { addr := &net.IPNet{ IP: deletedPodIP, - Mask: net.IPv4Mask(255, 255, 255, 255), + Mask: net.CIDRMask(maskLen, maskLen), } if r.PodVlanId != 0 { diff --git a/pkg/ipamd/rpc_handler.go b/pkg/ipamd/rpc_handler.go index b00cffcf9d..eb524fceaf 100644 --- a/pkg/ipamd/rpc_handler.go +++ b/pkg/ipamd/rpc_handler.go @@ -199,6 +199,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp log.Infof("Received DelNetwork for Sandbox %s", in.ContainerID) log.Debugf("DelNetworkRequest: %s", in) delIPCnt.With(prometheus.Labels{"reason": in.Reason}).Inc() + var ipv4Addr, ipv6Addr, cidrStr string // Do this early, but after logging trace if err := s.validateVersion(in.ClientVersion); err != nil { @@ -212,8 +213,14 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp NetworkName: in.NetworkName, } eni, ip, deviceNumber, err := s.ipamContext.dataStore.UnassignPodIPAddress(ipamKey) - cidr := net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)} - cidrStr := cidr.String() + if s.ipamContext.enableIPv4 { + ipv4Addr = ip + cidr := net.IPNet{IP: net.ParseIP(ip), Mask: net.IPv4Mask(255, 255, 255, 255)} + cidrStr = cidr.String() + } else if s.ipamContext.enableIPv6 { + ipv6Addr = ip + } + if s.ipamContext.enableIPv4 && eni != nil { //cidrStr will be pod IP i.e, IP/32 for v4 (or) IP/128 for v6. // Case 1: PD is enabled but IP/32 key in AvailableIPv4Cidrs[cidrStr] exists, this means it is a secondary IP. Added IsPrefix check just for sanity. @@ -256,7 +263,7 @@ func (s *server) DelNetwork(ctx context.Context, in *rpc.DelNetworkRequest) (*rp log.Infof("Send DelNetworkReply: IPv4Addr %s, DeviceNumber: %d, err: %v", ip, deviceNumber, err) - return &rpc.DelNetworkReply{Success: err == nil, IPv4Addr: ip, DeviceNumber: int32(deviceNumber)}, err + return &rpc.DelNetworkReply{Success: err == nil, IPv4Addr: ipv4Addr, IPv6Addr: ipv6Addr, DeviceNumber: int32(deviceNumber)}, err } // RunRPCHandler handles request from gRPC diff --git a/rpc/mocks/rpc_mocks.go b/rpc/mocks/rpc_mocks.go index 4211551cce..3e6d00cefc 100644 --- a/rpc/mocks/rpc_mocks.go +++ b/rpc/mocks/rpc_mocks.go @@ -20,11 +20,10 @@ package mock_rpc import ( context "context" - reflect "reflect" - rpc "github.com/aws/amazon-vpc-cni-k8s/rpc" gomock "github.com/golang/mock/gomock" grpc "google.golang.org/grpc" + reflect "reflect" ) // MockCNIBackendClient is a mock of CNIBackendClient interface diff --git a/rpc/rpc.pb.go b/rpc/rpc.pb.go index 02ac894713..eed86e2226 100644 --- a/rpc/rpc.pb.go +++ b/rpc/rpc.pb.go @@ -1,12 +1,17 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 +// protoc-gen-go v1.23.0 // protoc v3.17.2 // source: rpc.proto package rpc import ( + context "context" + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -20,6 +25,10 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + type AddNetworkRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -130,16 +139,16 @@ type AddNetworkReply struct { Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"` - IPv6Addr string `protobuf:"bytes,3,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"` + IPv6Addr string `protobuf:"bytes,11,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"` DeviceNumber int32 `protobuf:"varint,4,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"` UseExternalSNAT bool `protobuf:"varint,5,opt,name=UseExternalSNAT,proto3" json:"UseExternalSNAT,omitempty"` VPCv4CIDRs []string `protobuf:"bytes,6,rep,name=VPCv4CIDRs,proto3" json:"VPCv4CIDRs,omitempty"` + VPCv6CIDRs []string `protobuf:"bytes,12,rep,name=VPCv6CIDRs,proto3" json:"VPCv6CIDRs,omitempty"` // start of pod-eni parameters - PodVlanId int32 `protobuf:"varint,7,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` - PodENIMAC string `protobuf:"bytes,8,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` - PodENISubnetGW string `protobuf:"bytes,9,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` - ParentIfIndex int32 `protobuf:"varint,10,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` // end of pod-eni parameters - VPCv6CIDRs []string `protobuf:"bytes,11,rep,name=VPCv6CIDRs,proto3" json:"VPCv6CIDRs,omitempty"` + PodVlanId int32 `protobuf:"varint,7,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` + PodENIMAC string `protobuf:"bytes,8,opt,name=PodENIMAC,proto3" json:"PodENIMAC,omitempty"` + PodENISubnetGW string `protobuf:"bytes,9,opt,name=PodENISubnetGW,proto3" json:"PodENISubnetGW,omitempty"` + ParentIfIndex int32 `protobuf:"varint,10,opt,name=ParentIfIndex,proto3" json:"ParentIfIndex,omitempty"` // end of pod-eni parameters } func (x *AddNetworkReply) Reset() { @@ -216,6 +225,13 @@ func (x *AddNetworkReply) GetVPCv4CIDRs() []string { return nil } +func (x *AddNetworkReply) GetVPCv6CIDRs() []string { + if x != nil { + return x.VPCv6CIDRs + } + return nil +} + func (x *AddNetworkReply) GetPodVlanId() int32 { if x != nil { return x.PodVlanId @@ -244,13 +260,6 @@ func (x *AddNetworkReply) GetParentIfIndex() int32 { return 0 } -func (x *AddNetworkReply) GetVPCv6CIDRs() []string { - if x != nil { - return x.VPCv6CIDRs - } - return nil -} - type DelNetworkRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -361,6 +370,7 @@ type DelNetworkReply struct { Success bool `protobuf:"varint,1,opt,name=Success,proto3" json:"Success,omitempty"` IPv4Addr string `protobuf:"bytes,2,opt,name=IPv4Addr,proto3" json:"IPv4Addr,omitempty"` + IPv6Addr string `protobuf:"bytes,5,opt,name=IPv6Addr,proto3" json:"IPv6Addr,omitempty"` DeviceNumber int32 `protobuf:"varint,3,opt,name=DeviceNumber,proto3" json:"DeviceNumber,omitempty"` // start of pod-eni parameters PodVlanId int32 `protobuf:"varint,4,opt,name=PodVlanId,proto3" json:"PodVlanId,omitempty"` // end of pod-eni parameters @@ -412,6 +422,13 @@ func (x *DelNetworkReply) GetIPv4Addr() string { return "" } +func (x *DelNetworkReply) GetIPv6Addr() string { + if x != nil { + return x.IPv6Addr + } + return "" +} + func (x *DelNetworkReply) GetDeviceNumber() int32 { if x != nil { return x.DeviceNumber @@ -454,7 +471,7 @@ var file_rpc_proto_rawDesc = []byte{ 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, - 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x03, + 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, @@ -462,7 +479,9 @@ var file_rpc_proto_rawDesc = []byte{ 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x55, 0x73, 0x65, 0x45, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x53, 0x4e, 0x41, 0x54, 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x34, 0x43, 0x49, 0x44, 0x52, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x34, 0x43, 0x49, 0x44, 0x52, 0x73, 0x12, 0x1c, 0x0a, 0x09, + 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x34, 0x43, 0x49, 0x44, 0x52, 0x73, 0x12, 0x1e, 0x0a, 0x0a, + 0x56, 0x50, 0x43, 0x76, 0x36, 0x43, 0x49, 0x44, 0x52, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x36, 0x43, 0x49, 0x44, 0x52, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x4d, 0x41, 0x43, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x50, @@ -471,9 +490,7 @@ var file_rpc_proto_rawDesc = []byte{ 0x52, 0x0e, 0x50, 0x6f, 0x64, 0x45, 0x4e, 0x49, 0x53, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x47, 0x57, 0x12, 0x24, 0x0a, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, - 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1e, 0x0a, 0x0a, 0x56, 0x50, 0x43, 0x76, 0x36, 0x43, - 0x49, 0x44, 0x52, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x56, 0x50, 0x43, 0x76, - 0x36, 0x43, 0x49, 0x44, 0x52, 0x73, 0x22, 0xb7, 0x02, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x4e, 0x65, + 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0xb7, 0x02, 0x0a, 0x11, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, @@ -493,25 +510,29 @@ var file_rpc_proto_rawDesc = []byte{ 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x49, 0x66, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x4e, 0x61, 0x6d, 0x65, - 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, + 0x22, 0xa5, 0x01, 0x0a, 0x0f, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, - 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, - 0x0a, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x09, 0x50, 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x32, 0x88, 0x01, 0x0a, - 0x0a, 0x43, 0x4e, 0x49, 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3c, 0x0a, 0x0a, 0x41, - 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, - 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0a, 0x44, 0x65, 0x6c, - 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, - 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x14, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, - 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x42, 0x0a, 0x5a, 0x08, 0x72, 0x70, 0x63, 0x2f, 0x3b, - 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x08, 0x49, 0x50, 0x76, 0x34, 0x41, 0x64, 0x64, 0x72, 0x12, 0x1a, 0x0a, 0x08, 0x49, 0x50, + 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x49, 0x50, + 0x76, 0x36, 0x41, 0x64, 0x64, 0x72, 0x12, 0x22, 0x0a, 0x0c, 0x44, 0x65, 0x76, 0x69, 0x63, 0x65, + 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x44, 0x65, + 0x76, 0x69, 0x63, 0x65, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x50, 0x6f, + 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x50, + 0x6f, 0x64, 0x56, 0x6c, 0x61, 0x6e, 0x49, 0x64, 0x32, 0x88, 0x01, 0x0a, 0x0a, 0x43, 0x4e, 0x49, + 0x42, 0x61, 0x63, 0x6b, 0x65, 0x6e, 0x64, 0x12, 0x3c, 0x0a, 0x0a, 0x41, 0x64, 0x64, 0x4e, 0x65, + 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, + 0x72, 0x70, 0x63, 0x2e, 0x41, 0x64, 0x64, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3c, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x12, 0x16, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x6c, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x52, 0x65, 0x70, 0x6c, + 0x79, 0x22, 0x00, 0x42, 0x2b, 0x5a, 0x29, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x61, 0x77, 0x73, 0x2f, 0x61, 0x6d, 0x61, 0x7a, 0x6f, 0x6e, 0x2d, 0x76, 0x70, 0x63, + 0x2d, 0x63, 0x6e, 0x69, 0x2d, 0x6b, 0x38, 0x73, 0x2f, 0x72, 0x70, 0x63, 0x3b, 0x72, 0x70, 0x63, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -619,3 +640,119 @@ func file_rpc_proto_init() { file_rpc_proto_goTypes = nil file_rpc_proto_depIdxs = nil } + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// CNIBackendClient is the client API for CNIBackend service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type CNIBackendClient interface { + AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) + DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) +} + +type cNIBackendClient struct { + cc grpc.ClientConnInterface +} + +func NewCNIBackendClient(cc grpc.ClientConnInterface) CNIBackendClient { + return &cNIBackendClient{cc} +} + +func (c *cNIBackendClient) AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) { + out := new(AddNetworkReply) + err := c.cc.Invoke(ctx, "/rpc.CNIBackend/AddNetwork", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *cNIBackendClient) DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) { + out := new(DelNetworkReply) + err := c.cc.Invoke(ctx, "/rpc.CNIBackend/DelNetwork", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// CNIBackendServer is the server API for CNIBackend service. +type CNIBackendServer interface { + AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) + DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) +} + +// UnimplementedCNIBackendServer can be embedded to have forward compatible implementations. +type UnimplementedCNIBackendServer struct { +} + +func (*UnimplementedCNIBackendServer) AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method AddNetwork not implemented") +} +func (*UnimplementedCNIBackendServer) DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) { + return nil, status.Errorf(codes.Unimplemented, "method DelNetwork not implemented") +} + +func RegisterCNIBackendServer(s *grpc.Server, srv CNIBackendServer) { + s.RegisterService(&_CNIBackend_serviceDesc, srv) +} + +func _CNIBackend_AddNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AddNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CNIBackendServer).AddNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/rpc.CNIBackend/AddNetwork", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CNIBackendServer).AddNetwork(ctx, req.(*AddNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _CNIBackend_DelNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DelNetworkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CNIBackendServer).DelNetwork(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/rpc.CNIBackend/DelNetwork", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CNIBackendServer).DelNetwork(ctx, req.(*DelNetworkRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _CNIBackend_serviceDesc = grpc.ServiceDesc{ + ServiceName: "rpc.CNIBackend", + HandlerType: (*CNIBackendServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "AddNetwork", + Handler: _CNIBackend_AddNetwork_Handler, + }, + { + MethodName: "DelNetwork", + Handler: _CNIBackend_DelNetwork_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "rpc.proto", +} diff --git a/rpc/rpc.proto b/rpc/rpc.proto index 9fc2ca33b9..fbfb1ae760 100644 --- a/rpc/rpc.proto +++ b/rpc/rpc.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package rpc; -option go_package = "rpc/;rpc"; +option go_package = "github.com/aws/amazon-vpc-cni-k8s/rpc;rpc"; // The service definition. service CNIBackend { @@ -25,11 +25,11 @@ message AddNetworkRequest { message AddNetworkReply { bool Success = 1; string IPv4Addr = 2; - string IPv6Addr = 3; + string IPv6Addr = 11; int32 DeviceNumber = 4; bool UseExternalSNAT = 5; repeated string VPCv4CIDRs = 6; - + repeated string VPCv6CIDRs = 12; // start of pod-eni parameters int32 PodVlanId = 7; @@ -38,9 +38,7 @@ message AddNetworkReply { int32 ParentIfIndex = 10; // end of pod-eni parameters - repeated string VPCv6CIDRs = 11; - - // next field: 12 + // next field: 13 } message DelNetworkRequest { @@ -58,11 +56,12 @@ message DelNetworkRequest { message DelNetworkReply { bool Success = 1; string IPv4Addr = 2; + string IPv6Addr = 5; int32 DeviceNumber = 3; // start of pod-eni parameters int32 PodVlanId = 4; // end of pod-eni parameters - // next field: 5 + // next field: 6 } diff --git a/rpc/rpc_grpc.pb.go b/rpc/rpc_grpc.pb.go deleted file mode 100644 index 2c745f419d..0000000000 --- a/rpc/rpc_grpc.pb.go +++ /dev/null @@ -1,136 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. - -package rpc - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion6 - -// CNIBackendClient is the client API for CNIBackend service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type CNIBackendClient interface { - AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) - DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) -} - -type cNIBackendClient struct { - cc grpc.ClientConnInterface -} - -func NewCNIBackendClient(cc grpc.ClientConnInterface) CNIBackendClient { - return &cNIBackendClient{cc} -} - -func (c *cNIBackendClient) AddNetwork(ctx context.Context, in *AddNetworkRequest, opts ...grpc.CallOption) (*AddNetworkReply, error) { - out := new(AddNetworkReply) - err := c.cc.Invoke(ctx, "/rpc.CNIBackend/AddNetwork", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *cNIBackendClient) DelNetwork(ctx context.Context, in *DelNetworkRequest, opts ...grpc.CallOption) (*DelNetworkReply, error) { - out := new(DelNetworkReply) - err := c.cc.Invoke(ctx, "/rpc.CNIBackend/DelNetwork", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// CNIBackendServer is the server API for CNIBackend service. -// All implementations must embed UnimplementedCNIBackendServer -// for forward compatibility -type CNIBackendServer interface { - AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) - DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) -} - -// UnimplementedCNIBackendServer must be embedded to have forward compatible implementations. -type UnimplementedCNIBackendServer struct { -} - -func (UnimplementedCNIBackendServer) AddNetwork(context.Context, *AddNetworkRequest) (*AddNetworkReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method AddNetwork not implemented") -} -func (UnimplementedCNIBackendServer) DelNetwork(context.Context, *DelNetworkRequest) (*DelNetworkReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method DelNetwork not implemented") -} -func (UnimplementedCNIBackendServer) mustEmbedUnimplementedCNIBackendServer() {} - -// UnsafeCNIBackendServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to CNIBackendServer will -// result in compilation errors. -type UnsafeCNIBackendServer interface { - mustEmbedUnimplementedCNIBackendServer() -} - -func RegisterCNIBackendServer(s *grpc.Server, srv CNIBackendServer) { - s.RegisterService(&CNIBackend_ServiceDesc, srv) -} - -func _CNIBackend_AddNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AddNetworkRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CNIBackendServer).AddNetwork(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/rpc.CNIBackend/AddNetwork", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CNIBackendServer).AddNetwork(ctx, req.(*AddNetworkRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _CNIBackend_DelNetwork_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DelNetworkRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(CNIBackendServer).DelNetwork(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/rpc.CNIBackend/DelNetwork", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(CNIBackendServer).DelNetwork(ctx, req.(*DelNetworkRequest)) - } - return interceptor(ctx, in, info, handler) -} - -// CNIBackend_ServiceDesc is the grpc.ServiceDesc for CNIBackend service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var CNIBackend_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "rpc.CNIBackend", - HandlerType: (*CNIBackendServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "AddNetwork", - Handler: _CNIBackend_AddNetwork_Handler, - }, - { - MethodName: "DelNetwork", - Handler: _CNIBackend_DelNetwork_Handler, - }, - }, - Streams: []grpc.StreamDesc{}, - Metadata: "rpc/rpc.proto", -} From a74239e0fa499e1a6c9927764d7b1b7b2a5f315f Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Wed, 8 Sep 2021 11:52:59 -0700 Subject: [PATCH 4/7] Rearrange rpc proto fields + Log Cleanup --- cmd/routed-eni-cni-plugin/cni.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/routed-eni-cni-plugin/cni.go b/cmd/routed-eni-cni-plugin/cni.go index 562fef637a..cce3e04acd 100644 --- a/cmd/routed-eni-cni-plugin/cni.go +++ b/cmd/routed-eni-cni-plugin/cni.go @@ -334,7 +334,6 @@ func del(args *skel.CmdArgs, cniTypes typeswrapper.CNITYPES, grpcClient grpcwrap deletedPodIP = net.ParseIP(r.IPv4Addr) maskLen = 32 } else if r.IPv6Addr != "" { - log.Debug("Deleting Pod NS - V6 Mode") deletedPodIP = net.ParseIP(r.IPv6Addr) maskLen = 128 } From 74081c4643f66e5e091d2aaf0c7a843310bba849 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru <60630804+achevuru@users.noreply.github.com> Date: Sun, 19 Sep 2021 22:31:19 -0700 Subject: [PATCH 5/7] Update ipamd.go --- pkg/ipamd/ipamd.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 2b74233911..4505993480 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -1681,6 +1681,7 @@ func isIPv4Enabled() bool { func isIPv6Enabled() bool { return getEnvBoolWithDefault(envEnableIPv6, false) +} func enableManageUntaggedMode() bool { return getEnvBoolWithDefault(envManageUntaggedENI, true) From 90b77112aded24ce1ed51bf2e265d2a0468eb596 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Mon, 20 Sep 2021 10:18:58 -0700 Subject: [PATCH 6/7] Fix formatting and merge issues --- pkg/ipamd/datastore/data_store.go | 7 +++---- pkg/ipamd/ipamd.go | 30 +++++++++++++----------------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index 49d798cbd7..52582c2142 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -240,7 +240,7 @@ func (cidr *CidrInfo) AssignedIPAddressesInCidr() int { func (cidr *CidrInfo) GetIPStatsFromCidr() (int, int) { assignedIPs := 0 cooldownIPs := 0 - for _, addr := range cidr.IPv4Addresses { + for _, addr := range cidr.IPAddresses { if addr.Assigned() { assignedIPs++ } else if addr.inCoolingPeriod() { @@ -832,7 +832,6 @@ func (ds *DataStore) unassignPodIPAddressUnsafe(addr *AddressInfo) { assignedIPs.Set(float64(ds.assigned)) } - // GetStats returns total number of IP addresses, number of assigned IP addresses, total prefixes and IPs in cooldown period func (ds *DataStore) GetStats(addressFamily string) (int, int, int, int) { ds.lock.Lock() @@ -849,8 +848,8 @@ func (ds *DataStore) GetStats(addressFamily string) (int, int, int, int) { for _, cidr := range AssignedCIDRs { if addressFamily == "4" && ((ds.isPDEnabled && cidr.IsPrefix) || (!ds.isPDEnabled && !cidr.IsPrefix)) { assignedCount, cooldownCount := cidr.GetIPStatsFromCidr() - assignedIPs += assignedCount - cooldownIPs += cooldownCount + assignedIPs += assignedCount + cooldownIPs += cooldownCount totalIPs += cidr.Size() } else if addressFamily == "6" { assignedIPs += cidr.AssignedIPAddressesInCidr() diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 4505993480..61c99e9501 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -146,7 +146,7 @@ const ( ipV4AddrFamily = "4" ipV6AddrFamily = "6" - //insufficientCidrErrorCooldown is the amount of time reconciler will wait before trying to fetch + //insufficientCidrErrorCooldown is the amount of time reconciler will wait before trying to fetch //more IPs/prefixes for an ENI. With InsufficientCidr we know the subnet doesn't have enough IPs so //instead of retrying every 5s which would lead to increase in EC2 AllocIPAddress calls, we wait for //120 seconds for a retry. @@ -240,14 +240,14 @@ type IPAMContext struct { lastDecreaseIPPool time.Time // reconcileCooldownCache keeps timestamps of the last time an IP address was unassigned from an ENI, // so that we don't reconcile and add it back too quickly if IMDS lags behind reality. - reconcileCooldownCache ReconcileCooldownCache - terminating int32 // Flag to warn that the pod is about to shut down. - disableENIProvisioning bool - enablePodENI bool - myNodeName string - enablePrefixDelegation bool - lastInsufficientCidrError time.Time - enableManageUntaggedMode bool + reconcileCooldownCache ReconcileCooldownCache + terminating int32 // Flag to warn that the pod is about to shut down. + disableENIProvisioning bool + enablePodENI bool + myNodeName string + enablePrefixDelegation bool + lastInsufficientCidrError time.Time + enableManageUntaggedMode bool } // setUnmanagedENIs will rebuild the set of ENI IDs for ENIs tagged as "no_manage" @@ -377,12 +377,12 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex c.enablePodENI = enablePodENI() c.enableManageUntaggedMode = enableManageUntaggedMode() - err = c.awsClient.FetchInstanceTypeLimits() + err = c.awsClient.FetchInstanceTypeLimits() if err != nil { log.Errorf("Failed to get ENI limits from file:vpc_ip_limits or EC2 for %s", c.awsClient.GetInstanceType()) return nil, err } - + //Let's validate if the configured combination of env variables is supported before we //proceed any further if !c.isConfigValid() { @@ -1111,7 +1111,7 @@ func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv6Pref ipamdErrInc("addENIv6prefixesToDataStoreFailed") } } - _, _, totalPrefix := c.dataStore.GetStats(ipV6AddrFamily) + _, _, totalPrefix,_ := c.dataStore.GetStats(ipV6AddrFamily) log.Debugf("Datastore Pool stats: total v6 prefixes(/80): %d", totalPrefix) } @@ -2122,11 +2122,7 @@ func (c *IPAMContext) initENIAndIPLimits() (err error) { func (c *IPAMContext) isConfigValid() bool { //Get Instance type - hypervisorType, err := c.awsClient.GetInstanceHypervisorFamily() - if err != nil { - log.Error("Failed to get hypervisor type") - return false - } + hypervisorType := c.awsClient.GetInstanceHypervisorFamily() //Validate that only one among v4 and v6 is enabled. if c.enableIPv4 && c.enableIPv6 { From f7242d982b663fc86996374d8394cfef334e5c69 Mon Sep 17 00:00:00 2001 From: Apurup Chevuru Date: Mon, 20 Sep 2021 11:45:05 -0700 Subject: [PATCH 7/7] pkg/ipamd/ipamd.go --- pkg/ipamd/ipamd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index 61c99e9501..39b4382bab 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -1111,7 +1111,7 @@ func (c *IPAMContext) addENIv6prefixesToDataStore(ec2PrefixAddrs []*ec2.Ipv6Pref ipamdErrInc("addENIv6prefixesToDataStoreFailed") } } - _, _, totalPrefix,_ := c.dataStore.GetStats(ipV6AddrFamily) + _, _, totalPrefix, _ := c.dataStore.GetStats(ipV6AddrFamily) log.Debugf("Datastore Pool stats: total v6 prefixes(/80): %d", totalPrefix) }