From b16fdef39ac48e909b1ee2c9cc575f19c185dfb1 Mon Sep 17 00:00:00 2001 From: Levente Kale Date: Fri, 11 Oct 2019 15:22:45 +0200 Subject: [PATCH] Correcting UT, updating documentation --- pkg/cnidel/cniconfs.go | 1 - test/utils/utils.go | 34 ++++++++++++++++------------- test/uts/cnidel_test/cnidel_test.go | 11 +++++----- user-guide.md | 15 ++++++++++++- 4 files changed, 39 insertions(+), 22 deletions(-) diff --git a/pkg/cnidel/cniconfs.go b/pkg/cnidel/cniconfs.go index c41790c7..bd927061 100644 --- a/pkg/cnidel/cniconfs.go +++ b/pkg/cnidel/cniconfs.go @@ -62,7 +62,6 @@ func getSriovCniConfig(netInfo *danmtypes.DanmNet, ipamOptions datastructs.IpamC sriovConfig.CNIVersion = cniVersion sriovConfig.Name = netInfo.Spec.NetworkID sriovConfig.Type = "sriov" - // initialize SriovNet specific fields: pfname, err := sriov_utils.GetPfName(ep.Spec.Iface.DeviceID) if err != nil { return nil, errors.New("failed to get the name of the sriov PF for device "+ ep.Spec.Iface.DeviceID +" due to:" + err.Error()) diff --git a/test/utils/utils.go b/test/utils/utils.go index 6f4f44ee..6659937d 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -58,26 +58,30 @@ type MalformedObject struct { func SetupAllocationPools(nets []danmtypes.DanmNet) error { for index, dnet := range nets { if dnet.Spec.Options.Cidr != "" { - admit.CreateAllocationArray(&dnet) - _, ipnet, err := net.ParseCIDR(dnet.Spec.Options.Cidr) - if err != nil { - return err - } - if dnet.Spec.Options.Pool.Start == "" { - dnet.Spec.Options.Pool.Start = (ipam.Int2ip(ipam.Ip2int(ipnet.IP) + 1)).String() - } - if dnet.Spec.Options.Pool.End == "" { - dnet.Spec.Options.Pool.End = (ipam.Int2ip(ipam.Ip2int(admit.GetBroadcastAddress(ipnet)) - 1)).String() - } - if strings.HasPrefix(dnet.ObjectMeta.Name, "full") { - exhaustNetwork(&dnet) - } - nets[index].Spec = dnet.Spec + nets[index].Spec = InitAllocPool(&dnet).Spec } } return nil } +func InitAllocPool(dnet *danmtypes.DanmNet) *danmtypes.DanmNet { + if dnet.Spec.Options.Cidr == "" { + return dnet + } + admit.CreateAllocationArray(dnet) + _, ipnet, _ := net.ParseCIDR(dnet.Spec.Options.Cidr) + if dnet.Spec.Options.Pool.Start == "" { + dnet.Spec.Options.Pool.Start = (ipam.Int2ip(ipam.Ip2int(ipnet.IP) + 1)).String() + } + if dnet.Spec.Options.Pool.End == "" { + dnet.Spec.Options.Pool.End = (ipam.Int2ip(ipam.Ip2int(admit.GetBroadcastAddress(ipnet)) - 1)).String() + } + if strings.HasPrefix(dnet.ObjectMeta.Name, "full") { + exhaustNetwork(dnet) + } + return dnet +} + func GetTestNet(netId string, testNets []danmtypes.DanmNet) *danmtypes.DanmNet { for _, net := range testNets { if net.ObjectMeta.Name == netId { diff --git a/test/uts/cnidel_test/cnidel_test.go b/test/uts/cnidel_test/cnidel_test.go index b15ccd07..da9f671c 100644 --- a/test/uts/cnidel_test/cnidel_test.go +++ b/test/uts/cnidel_test/cnidel_test.go @@ -134,10 +134,10 @@ var expectedCniConfigs = []CniConf { {"macvlan-ip4", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"macvlan-ip6", []byte(`{"cniexp":{"cnitype":"macvlan","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v6","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam"}}}`)}, {"macvlan-dual-stack", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-ds","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, - {"macvlan-ip4-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.66/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.66/26","version":4}]}}}`)}, + {"macvlan-ip4-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"macvlan-ip6-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v6","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam"}}}`)}, - {"sriov-l3", []byte(`{"cniexp":{"cnitype":"sriov","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","l2enable":false,"vlan":500,"deviceID":"0000:af:06.0","ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, - {"sriov-l2", []byte(`{"cniexp":{"cnitype":"sriov","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","l2enable":true,"vlan":500,"deviceID":"0000:af:06.0"}}`)}, + {"sriov-l3", []byte(`{"cniexp":{"cnitype":"sriov","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","vlan":500,"deviceID":"0000:af:06.0","ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, + {"sriov-l2", []byte(`{"cniexp":{"cnitype":"sriov","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","vlan":500,"deviceID":"0000:af:06.0"}}`)}, {"deleteflannel", []byte(`{"cniexp":{"cnitype":"flannel","env":{"CNI_COMMAND":"DEL","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"cbr0","type":"flannel","delegate":{"hairpinMode":true,"isDefaultGateway":true}}}`)}, {"deletemacvlan", []byte(`{"cniexp":{"cnitype":"macvlan","env":{"CNI_COMMAND":"DEL","CNI_IFNAME":"ens1f0"}},"cniconf":{"cniVersion":"0.3.1","name":"full","master":"ens1f0","mode":"bridge","mtu":1500,"ipam": {"type": "fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"bridge-l3-ip4", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name": "mynet","type": "bridge","bridge": "mynet0","isDefaultGateway": true,"forceAddress": false,"ipMasq": true,"hairpinMode": true,"ipam": {"type": "fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, @@ -253,12 +253,12 @@ var delSetupTcs = []struct { {"dynamicMacvlanDualStack", "macvlan-ds", "dynamicDual", "macvlan-dual-stack", "192.168.1.65", "2a00:8a00:a000:1193", false, 1}, {"dynamicMacvlanIpv4Type020Result", "macvlan-v4", "dynamicIpv4", "macvlan-ip4-type020", "192.168.1.65", "", false, 1}, {"dynamicMacvlanIpv6Type020Result", "macvlan-v6", "dynamicIpv6", "macvlan-ip6-type020", "", "2a00:8a00:a000:1193", false, 0}, - {"dynamicSriovNoDeviceId", "sriov-test", "dynamicIpv4", "", "", "", true, 2}, + {"dynamicSriovNoDeviceId", "sriov-test", "dynamicIpv4", "", "", "", true, 1}, {"dynamicSriovL3", "sriov-test", "dynamicIpv4WithDeviceId", "sriov-l3", "", "", false, 1}, {"dynamicSriovL2", "sriov-test", "noneWithDeviceId", "sriov-l2", "", "", false, 0}, {"bridgeWithV4Overwrite", "bridge-ipam-ipv4", "simpleIpv4", "bridge-l3-ip4", "", "", false, 1}, {"bridgeWithV4Add", "bridge-ipam-l2", "simpleIpv4", "bridge-l2-ip4", "", "", false, 1}, - {"bridgeWithInvalidAdd", "bridge-invalid", "simpleIpv4", "", "", "", true, 2}, + {"bridgeWithInvalidAdd", "bridge-invalid", "simpleIpv4", "", "", "", true, 1}, {"bridgeL3OriginalNoCidr", "bridge-noipam", "simpleIpv4", "bridge-l3-orig", "", "", false, 0}, {"bridgeL3OriginalNoIp", "bridge-ipam-ipv4", "noIps", "bridge-l3-orig", "", "", false, 0}, {"bridgeL2OriginalNoCidr", "bridge-noipam-l2", "simpleIpv4", "bridge-l2-orig", "", "", false, 0}, @@ -358,6 +358,7 @@ func TestDelegateInterfaceSetup(t *testing.T) { testNet := utils.GetTestNet(tc.netName, testNets) testEp := getTestEp(tc.epName) testEp.Spec.NetworkName = testNet.ObjectMeta.Name + testNet = utils.InitAllocPool(testNet) cniRes, err := cnidel.DelegateInterfaceSetup(&cniConf,netClientStub,testNet,testEp) if (err != nil && !tc.isErrorExpected) || (err == nil && tc.isErrorExpected) { var detailedErrorMessage string diff --git a/user-guide.md b/user-guide.md index 7f88324c..b8a8dc80 100644 --- a/user-guide.md +++ b/user-guide.md @@ -23,6 +23,7 @@ * [DANM IPVLAN CNI](#danm-ipvlan-cni) * [Device Plugin Support](#device-plugin-support) * [Using Intel SR-IOV CNI](#using-intel-sr-iov-cni) + * [DPDK support](#dpdk-support) * [Usage of DANM's Webhook component](#usage-of-danms-webhook-component) * [Responsibilities](#responsibilities) * [Connecting TenantNetworks to TenantConfigs](#connecting-tenantnetworks-to-tenantconfigs) @@ -259,7 +260,8 @@ The CNI provisions IPVLAN interfaces in L2 mode, and supports the following extr * provisioning generic IP routes into a configured routing table inside the Pod's network namespace * Pod-level controlled provisioning of policy-based IP routes into Pod's network namespace #### Device Plugin support -DANM provides general support to CNIs which interwork with Kubernetes' Device Plugin mechanism such as SR-IOV CNI. +DANM provides general support for CNIs interworking with Kubernetes' Device Plugin mechanism. +A practical example of such a network provisioner is the SR-IOV CNI. When a properly configured Network Device Plugin runs, the allocatable resource list for the node should be updated with resource discovered by the plugin. ##### Using Intel SR-IOV CNI SR-IOV Network Device Plugin allows to create a list of *netdevice* type resource definitions with *sriovMode*, where each resource definition can have one or more assigned *rootDevice* (Physical Function). The plugin looks for Virtual Functions (VF) for each configured Physical Function (PF) and adds all discovered VFs to the allocatable resource's list of the given Kubernetes Node. The Device Plugin resource name will be the device pool name on the Node. These device pools can be referred in Pod definition's resource request part on the usual way. @@ -329,6 +331,17 @@ spec: nodeSelector: sriov: enabled ``` +##### DPDK support +DANM's SR-IOV integration supports -and is tested with- both Intel, and Mellanox manufactured physical functions. +Moreover Pods can use the allocated Virtual Functions for either kernel, or userspace networking. +The only restriction to keep in mind is when a DPDK using application requests VFs from an Intel NIC for the purpose of userspace networking, +those VFs shall be already bound to the vfio-pci kernel driver before the Pod is instantiated. +To guarantee such VFs are always available on the Node the Pod is scheduled to, we strongly suggest advertising vfio-pci bound VFs as a separate Device Pool. +When an already vfio bound function is mounted to an application, DANM also creates a dummy kernel interface in its stead in the Pod's network namespace. +The dummy interface can be easily identified by the application, because it's named exactly as the VF would be. +The dummy interface is used to communicate the IPAM details belonging to the non-kernel managed device, such as IP addresses, IP routes etc. +Userspace applications can interrogate this information via the usual kernel APIs, and then configure the allocated resources into their own IP stack! + ### Usage of DANM's Webhook component #### Responsibilities The Webhook component introduced in DANM V4 is responsible for three things: