diff --git a/crd/apis/danm/v1/types.go b/crd/apis/danm/v1/types.go index 870dfc66..5954db10 100644 --- a/crd/apis/danm/v1/types.go +++ b/crd/apis/danm/v1/types.go @@ -87,7 +87,7 @@ type DanmEpIface struct { MacAddress string `json:"MacAddress"` Proutes map[string]string `json:"proutes"` Proutes6 map[string]string `json:"proutes6"` - DeviceID string `json:"DeviceID,omitempty"` + DeviceID string `json:"DeviceID,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/glide.lock b/glide.lock index b0a5fe53..253c6e24 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 1d7eea7c8fe94fb460fead10727227389ddead184b862425f9a6a913ec4b6e94 -updated: 2019-08-06T22:31:42.529643558+02:00 +hash: aa386d47de4210547ee89573057b83178ae6f7f4b1dd1df33f0cde11cf69b228 +updated: 2019-09-27T17:53:22.774886331+02:00 imports: - name: github.com/apparentlymart/go-cidr version: 2bd8b58cf4275aeb086ade613de226773e29e853 @@ -72,8 +72,9 @@ imports: - logging - types - name: github.com/intel/sriov-cni - version: 9e4c973b2ac517c64867e33d61aee152d70dc330 + version: 365c8f8cc1204df84f3e976ea30f113e733ca665 subpackages: + - pkg/types - pkg/utils - name: github.com/j-keck/arping version: 2cf9dc699c5640a7e2c81403a44127bf28033600 diff --git a/glide.yaml b/glide.yaml index e11bbfa8..470cdead 100644 --- a/glide.yaml +++ b/glide.yaml @@ -19,6 +19,5 @@ import: version: v0.8.1 - package: github.com/intel/multus-cni version: e723aabca8792ddd181167660e252a127a81b073 -- package: github.com/intel/sriov-cni - version: 9e4c973b2ac517c64867e33d61aee152d70dc330 - +- package: github.com/intel/sriov-cni + version: 365c8f8cc1204df84f3e976ea30f113e733ca665 diff --git a/pkg/cnidel/cniconfs.go b/pkg/cnidel/cniconfs.go index 4c761417..bd927061 100644 --- a/pkg/cnidel/cniconfs.go +++ b/pkg/cnidel/cniconfs.go @@ -62,18 +62,15 @@ func getSriovCniConfig(netInfo *danmtypes.DanmNet, ipamOptions datastructs.IpamC sriovConfig.CNIVersion = cniVersion sriovConfig.Name = netInfo.Spec.NetworkID sriovConfig.Type = "sriov" - // initialize SriovNet specific fields: pfname, err := sriov_utils.GetPfName(ep.Spec.Iface.DeviceID) if err != nil { return nil, errors.New("failed to get the name of the sriov PF for device "+ ep.Spec.Iface.DeviceID +" due to:" + err.Error()) } - sriovConfig.PfName = pfname - sriovConfig.L2Mode = true + sriovConfig.Master = pfname sriovConfig.Vlan = netInfo.Spec.Options.Vlan sriovConfig.DeviceID = ep.Spec.Iface.DeviceID if len(ipamOptions.Ips) > 0 { sriovConfig.Ipam = ipamOptions - sriovConfig.L2Mode = false } rawConfig, err := json.Marshal(sriovConfig) if err != nil { diff --git a/pkg/cnidel/cnidel.go b/pkg/cnidel/cnidel.go index 03eb9eab..bca1e10b 100644 --- a/pkg/cnidel/cnidel.go +++ b/pkg/cnidel/cnidel.go @@ -13,7 +13,6 @@ import ( "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" - "github.com/nokia/danm/pkg/danmep" "github.com/nokia/danm/pkg/datastructs" "github.com/nokia/danm/pkg/ipam" "github.com/nokia/danm/pkg/netcontrol" @@ -58,28 +57,23 @@ func DelegateInterfaceSetup(netConf *datastructs.NetConf, danmClient danmclients //As netInfo is only copied to IPAM above, the IP allocation is not refreshed in the original copy. //Without re-reading the network body we risk leaking IPs if error happens later on within the same thread! netInfo,err = netcontrol.GetNetworkFromEp(danmClient, *ep) - if err != nil {log.Println("lofasz:" + err.Error())} + if err != nil { + return nil, err + } ipamOptions = getCniIpamConfig(netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) } rawConfig, err := getCniPluginConfig(netConf, netInfo, ipamOptions, ep) if err != nil { - FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) return nil, err } cniType := netInfo.Spec.NetworkType cniResult,err := execCniPlugin(cniType, CniAddOp, netInfo, rawConfig, ep) if err != nil { - FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) return nil, errors.New("Error delegating ADD to CNI plugin:" + cniType + " because:" + err.Error()) } if cniResult != nil { setEpIfaceAddress(cniResult, &ep.Spec.Iface) } - err = danmep.CreateRoutesInNetNs(*ep, netInfo) - if err != nil { - // We don't consider this serious error, so we only log a warning about the issue. - log.Println("WARNING: Could not create IP routes for CNI:" + cniType + " because:" + err.Error()) - } return cniResult, nil } diff --git a/pkg/cnidel/types.go b/pkg/cnidel/types.go index a4db42d5..e820b19e 100644 --- a/pkg/cnidel/types.go +++ b/pkg/cnidel/types.go @@ -2,6 +2,7 @@ package cnidel import ( "github.com/containernetworking/cni/pkg/types" + sriov_types "github.com/intel/sriov-cni/pkg/types" danmtypes "github.com/nokia/danm/crd/apis/danm/v1" "github.com/nokia/danm/pkg/datastructs" ) @@ -17,24 +18,9 @@ type cniBackendConfig struct { // sriovNet represent the configuration of sriov cni v1.0.0 type SriovNet struct { - types.NetConf - // name of the PF since sriov cni v1.0.0 - PfName string `json:"master"` - // if true then add VF as L2 mode only, IPAM will not be executed - L2Mode bool `json:"l2enable,omitEmpty"` - // VLAN ID to assign for the VF - Vlan int `json:"vlan,omitEmpty"` + sriov_types.NetConf // IPAM configuration to be used for this network Ipam datastructs.IpamConfig `json:"ipam,omitEmpty"` - // Device PCI ID - DeviceID string `json:"deviceID"` -} - -// VfInformation is a DeviceInfo descriptor expected by sriov cni v1.0.0 -type VfInformation struct { - PCIaddr string `json:"pci_addr"` - Pfname string `json:"pfname"` - Vfid int `json:"vfid"` } type MacvlanNet struct { diff --git a/pkg/danmep/danmep.go b/pkg/danmep/danmep.go index 52bf8669..a07ddf95 100644 --- a/pkg/danmep/danmep.go +++ b/pkg/danmep/danmep.go @@ -10,6 +10,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils/sysctl" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + sriov_utils "github.com/intel/sriov-cni/pkg/utils" danmtypes "github.com/nokia/danm/crd/apis/danm/v1" danmclientset "github.com/nokia/danm/crd/client/clientset/versioned" ) @@ -107,7 +108,7 @@ func DetermineHostDeviceName(dnet *danmtypes.DanmNet) string { return device } -func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { +func PostProcessInterface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { runtime.LockOSThread() defer runtime.UnlockOSThread() origNs, err := ns.GetCurrentNS() @@ -127,33 +128,24 @@ func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { }() err = hns.Set() if err != nil { - return errors.New("failed to enter network namespace of CID:"+ep.Spec.Netns+" with error:"+err.Error()) + return errors.New("failed to enter network namespace of CID:" + ep.Spec.Netns + " with error:" + err.Error()) } - return addIpRoutes(ep,dnet) -} - -func SetDanmEpSysctls(ep danmtypes.DanmEp) error { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - // save the current namespace - origNs, err := ns.GetCurrentNS() - if err != nil { - return errors.New("failed to get current network namespace due to:" + err.Error()) - } - // enter to the Pod's network namespace - podNs, err := ns.GetNS(ep.Spec.Netns) - if err != nil { - return errors.New("failed to get Pod's network namespace due to:" + err.Error()) + isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID) + if isVfAttachedToDpdkDriver { + err = createDummyInterface(ep) + if err != nil { + return errors.New("failed to create dummy kernel interface for " + ep.Spec.Iface.Name + " because:" + err.Error()) + } } - err = podNs.Set() + err = setDanmEpSysctls(ep) if err != nil { - return errors.New("failed to switch to Pod's network namespace due to:" + err.Error()) + return errors.New("failed to set kernel configs for interface" + ep.Spec.Iface.Name + " beause:" + err.Error()) } - defer func() { - podNs.Close() - origNs.Set() - }() - // set sysctls + return addIpRoutes(ep,dnet) +} + +func setDanmEpSysctls(ep danmtypes.DanmEp) error { + var err error for _, s := range sysctls { if s.sysctlFunc(ep) { for _, ss := range s.sysctlData { diff --git a/pkg/danmep/ep.go b/pkg/danmep/ep.go index c70b6239..fe765349 100644 --- a/pkg/danmep/ep.go +++ b/pkg/danmep/ep.go @@ -29,12 +29,7 @@ func createIpvlanInterface(dnet *danmtypes.DanmNet, ep danmtypes.DanmEp) error { return createContainerIface(ep, dnet, device) } -// TODO: Refactor this, as cyclomatic complexity is too high func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device string) error { - var ( - addr4, addr6 net.IP - pref4, pref6 *net.IPNet - ) runtime.LockOSThread() defer runtime.UnlockOSThread() origns, err := ns.GetCurrentNS() @@ -87,49 +82,54 @@ func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device s if err != nil { return errors.New("cannot find IPVLAN interface in network namespace:" + err.Error()) } - ip := ep.Spec.Iface.Address - if ip != "" { - addr4, pref4, err = net.ParseCIDR(ip) - if err != nil { - return errors.New("cannot parse ip4 address because:" + err.Error()) - } - ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr4, Mask: pref4.Mask}} - err = netlink.AddrAdd(iface, ipAddr) + err = configureLink(iface, ep) + if err != nil { + return err + } + if ep.Spec.Iface.Address != "" { + addr,_,_ := net.ParseCIDR(ep.Spec.Iface.Address) + err = arping.GratuitousArpOverIfaceByName(addr, ep.Spec.Iface.Name) if err != nil { - return errors.New("Cannot add ip4 address to IPVLAN interface because:" + err.Error()) + log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!") } } - ip6 := ep.Spec.Iface.AddressIPv6 - // TODO: Refactor, duplicate of 87-98 - if ip6 != "" { - addr6, pref6, err = net.ParseCIDR(ip6) + return nil +} + +func configureLink(iface netlink.Link, ep danmtypes.DanmEp) error { + var err error + if ep.Spec.Iface.Address != "" { + err = addIpToLink(ep.Spec.Iface.Address, iface) if err != nil { - return errors.New("cannot parse ip6 address because:" + err.Error()) + return err } - ipAddr6 := &netlink.Addr{IPNet: &net.IPNet{IP: addr6, Mask: pref6.Mask}} - err = netlink.AddrAdd(iface, ipAddr6) + } + if ep.Spec.Iface.AddressIPv6 != "" { + err = addIpToLink(ep.Spec.Iface.AddressIPv6, iface) if err != nil { - return errors.New("Cannot add ip6 address to IPVLAN interface because:" + err.Error()) + return err } } - dstPrefix := ep.Spec.Iface.Name - err = netlink.LinkSetName(iface, dstPrefix) + err = netlink.LinkSetName(iface, ep.Spec.Iface.Name) if err != nil { - return errors.New("cannot rename IPVLAN interface because:" + err.Error()) + return errors.New("cannot rename link:" + ep.Spec.Iface.Name + " because:" + err.Error()) } err = netlink.LinkSetUp(iface) if err != nil { - return errors.New("cannot set renamed IPVLAN interface to up because:" + err.Error()) + return errors.New("cannot set link:" + ep.Spec.Iface.Name + " UP because:" + err.Error()) } - if ip != "" { - err = arping.GratuitousArpOverIfaceByName(addr4, dstPrefix) - if err != nil { - log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!") - } + return nil +} + +func addIpToLink(ip string, iface netlink.Link) error { + addr, pref, err := net.ParseCIDR(ip) + if err != nil { + return errors.New("cannot parse IP address because:" + err.Error()) } - err = addIpRoutes(ep, dnet) + ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr, Mask: pref.Mask}} + err = netlink.AddrAdd(iface, ipAddr) if err != nil { - return errors.New("IP routes could not be provisioned, because:" + err.Error()) + return errors.New("cannot add IP address to link because:" + err.Error()) } return nil } @@ -212,7 +212,7 @@ func deleteContainerIface(ep danmtypes.DanmEp) error { defer runtime.UnlockOSThread() origns, err := ns.GetCurrentNS() if err != nil { - return errors.New("getting the current netNS failed") + return errors.New("getting the current netNS failed") } hns, err := ns.GetNS(ep.Spec.Netns) if err != nil { @@ -259,3 +259,20 @@ func deleteEp(ep danmtypes.DanmEp) error { } return deleteContainerIface(ep) } + +func createDummyInterface(ep danmtypes.DanmEp) error { + dummy := &netlink.Dummy { + LinkAttrs: netlink.LinkAttrs { + Name: ep.Spec.Iface.Name, + }, + } + err := netlink.LinkAdd(dummy) + if err != nil { + return errors.New("cannot create dummy interface for DPDK because:" + err.Error()) + } + iface, err := netlink.LinkByName(ep.Spec.Iface.Name) + if err != nil { + return errors.New("cannot find freshly created dummy interface because:" + err.Error()) + } + return configureLink(iface, ep) +} \ No newline at end of file diff --git a/pkg/metacni/metacni.go b/pkg/metacni/metacni.go index bbe754ed..86430ccf 100644 --- a/pkg/metacni/metacni.go +++ b/pkg/metacni/metacni.go @@ -369,19 +369,20 @@ func createDelegatedInterface(syncher *syncher.Syncher, danmClient danmclientset } delegatedResult,err := cnidel.DelegateInterfaceSetup(DanmConfig, danmClient, netInfo, &ep) if err != nil { + cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("CNI delegation failed due to error:" + err.Error()), nil) return } - err = danmep.PutDanmEp(danmClient, ep) + err = danmep.PostProcessInterface(ep, netInfo) if err != nil { cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) - syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("DanmEp object could not be PUT to K8s due to error:" + err.Error()), nil) + syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("Post-processing failed for interface:" + ep.Spec.Iface.Name + " because:" + err.Error()), nil) return } - err = danmep.SetDanmEpSysctls(ep) + err = danmep.PutDanmEp(danmClient, ep) if err != nil { cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6) - syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("Sysctls could not be set due to error:" + err.Error()), nil) + syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("DanmEp object could not be PUT to K8s due to error:" + err.Error()), nil) return } syncher.PushResult(netInfo.ObjectMeta.Name, nil, delegatedResult) @@ -423,7 +424,7 @@ func createDanmInterface(syncher *syncher.Syncher, danmClient danmclientset.Inte syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("IPVLAN interface could not be created due to error:" + err.Error()), nil) return } - err = danmep.SetDanmEpSysctls(ep) + err = danmep.PostProcessInterface(ep, netInfo) if err != nil { ipam.GarbageCollectIps(danmClient, netInfo, ip4, ip6) danmClient.DanmV1().DanmEps(ep.ObjectMeta.Namespace).Delete(ep.ObjectMeta.Name, &meta_v1.DeleteOptions{}) diff --git a/test/utils/utils.go b/test/utils/utils.go index 6f4f44ee..6659937d 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -58,26 +58,30 @@ type MalformedObject struct { func SetupAllocationPools(nets []danmtypes.DanmNet) error { for index, dnet := range nets { if dnet.Spec.Options.Cidr != "" { - admit.CreateAllocationArray(&dnet) - _, ipnet, err := net.ParseCIDR(dnet.Spec.Options.Cidr) - if err != nil { - return err - } - if dnet.Spec.Options.Pool.Start == "" { - dnet.Spec.Options.Pool.Start = (ipam.Int2ip(ipam.Ip2int(ipnet.IP) + 1)).String() - } - if dnet.Spec.Options.Pool.End == "" { - dnet.Spec.Options.Pool.End = (ipam.Int2ip(ipam.Ip2int(admit.GetBroadcastAddress(ipnet)) - 1)).String() - } - if strings.HasPrefix(dnet.ObjectMeta.Name, "full") { - exhaustNetwork(&dnet) - } - nets[index].Spec = dnet.Spec + nets[index].Spec = InitAllocPool(&dnet).Spec } } return nil } +func InitAllocPool(dnet *danmtypes.DanmNet) *danmtypes.DanmNet { + if dnet.Spec.Options.Cidr == "" { + return dnet + } + admit.CreateAllocationArray(dnet) + _, ipnet, _ := net.ParseCIDR(dnet.Spec.Options.Cidr) + if dnet.Spec.Options.Pool.Start == "" { + dnet.Spec.Options.Pool.Start = (ipam.Int2ip(ipam.Ip2int(ipnet.IP) + 1)).String() + } + if dnet.Spec.Options.Pool.End == "" { + dnet.Spec.Options.Pool.End = (ipam.Int2ip(ipam.Ip2int(admit.GetBroadcastAddress(ipnet)) - 1)).String() + } + if strings.HasPrefix(dnet.ObjectMeta.Name, "full") { + exhaustNetwork(dnet) + } + return dnet +} + func GetTestNet(netId string, testNets []danmtypes.DanmNet) *danmtypes.DanmNet { for _, net := range testNets { if net.ObjectMeta.Name == netId { diff --git a/test/uts/cnidel_test/cnidel_test.go b/test/uts/cnidel_test/cnidel_test.go index b15ccd07..da9f671c 100644 --- a/test/uts/cnidel_test/cnidel_test.go +++ b/test/uts/cnidel_test/cnidel_test.go @@ -134,10 +134,10 @@ var expectedCniConfigs = []CniConf { {"macvlan-ip4", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"macvlan-ip6", []byte(`{"cniexp":{"cnitype":"macvlan","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v6","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam"}}}`)}, {"macvlan-dual-stack", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"}},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-ds","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, - {"macvlan-ip4-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.66/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.66/26","version":4}]}}}`)}, + {"macvlan-ip4-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f0"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v4","master":"ens1f0","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"macvlan-ip6-type020", []byte(`{"cniexp":{"cnitype":"macvlan","ip6":"2a00:8a00:a000:1193::/64","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"ens1f1"},"return":"020"},"cniconf":{"cniVersion":"0.3.1","name":"macvlan-v6","master":"ens1f1","mode":"bridge","mtu":1500,"ipam":{"type":"fakeipam"}}}`)}, - {"sriov-l3", []byte(`{"cniexp":{"cnitype":"sriov","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","l2enable":false,"vlan":500,"deviceID":"0000:af:06.0","ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, - {"sriov-l2", []byte(`{"cniexp":{"cnitype":"sriov","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","l2enable":true,"vlan":500,"deviceID":"0000:af:06.0"}}`)}, + {"sriov-l3", []byte(`{"cniexp":{"cnitype":"sriov","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","vlan":500,"deviceID":"0000:af:06.0","ipam":{"type":"fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, + {"sriov-l2", []byte(`{"cniexp":{"cnitype":"sriov","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"sriov-test","type":"sriov","master":"enp175s0f1","vlan":500,"deviceID":"0000:af:06.0"}}`)}, {"deleteflannel", []byte(`{"cniexp":{"cnitype":"flannel","env":{"CNI_COMMAND":"DEL","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name":"cbr0","type":"flannel","delegate":{"hairpinMode":true,"isDefaultGateway":true}}}`)}, {"deletemacvlan", []byte(`{"cniexp":{"cnitype":"macvlan","env":{"CNI_COMMAND":"DEL","CNI_IFNAME":"ens1f0"}},"cniconf":{"cniVersion":"0.3.1","name":"full","master":"ens1f0","mode":"bridge","mtu":1500,"ipam": {"type": "fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, {"bridge-l3-ip4", []byte(`{"cniexp":{"cnitype":"macvlan","ip":"192.168.1.65/26","env":{"CNI_COMMAND":"ADD","CNI_IFNAME":"eth0"}},"cniconf":{"cniVersion":"0.3.1","name": "mynet","type": "bridge","bridge": "mynet0","isDefaultGateway": true,"forceAddress": false,"ipMasq": true,"hairpinMode": true,"ipam": {"type": "fakeipam","ips":[{"ipcidr":"192.168.1.65/26","version":4}]}}}`)}, @@ -253,12 +253,12 @@ var delSetupTcs = []struct { {"dynamicMacvlanDualStack", "macvlan-ds", "dynamicDual", "macvlan-dual-stack", "192.168.1.65", "2a00:8a00:a000:1193", false, 1}, {"dynamicMacvlanIpv4Type020Result", "macvlan-v4", "dynamicIpv4", "macvlan-ip4-type020", "192.168.1.65", "", false, 1}, {"dynamicMacvlanIpv6Type020Result", "macvlan-v6", "dynamicIpv6", "macvlan-ip6-type020", "", "2a00:8a00:a000:1193", false, 0}, - {"dynamicSriovNoDeviceId", "sriov-test", "dynamicIpv4", "", "", "", true, 2}, + {"dynamicSriovNoDeviceId", "sriov-test", "dynamicIpv4", "", "", "", true, 1}, {"dynamicSriovL3", "sriov-test", "dynamicIpv4WithDeviceId", "sriov-l3", "", "", false, 1}, {"dynamicSriovL2", "sriov-test", "noneWithDeviceId", "sriov-l2", "", "", false, 0}, {"bridgeWithV4Overwrite", "bridge-ipam-ipv4", "simpleIpv4", "bridge-l3-ip4", "", "", false, 1}, {"bridgeWithV4Add", "bridge-ipam-l2", "simpleIpv4", "bridge-l2-ip4", "", "", false, 1}, - {"bridgeWithInvalidAdd", "bridge-invalid", "simpleIpv4", "", "", "", true, 2}, + {"bridgeWithInvalidAdd", "bridge-invalid", "simpleIpv4", "", "", "", true, 1}, {"bridgeL3OriginalNoCidr", "bridge-noipam", "simpleIpv4", "bridge-l3-orig", "", "", false, 0}, {"bridgeL3OriginalNoIp", "bridge-ipam-ipv4", "noIps", "bridge-l3-orig", "", "", false, 0}, {"bridgeL2OriginalNoCidr", "bridge-noipam-l2", "simpleIpv4", "bridge-l2-orig", "", "", false, 0}, @@ -358,6 +358,7 @@ func TestDelegateInterfaceSetup(t *testing.T) { testNet := utils.GetTestNet(tc.netName, testNets) testEp := getTestEp(tc.epName) testEp.Spec.NetworkName = testNet.ObjectMeta.Name + testNet = utils.InitAllocPool(testNet) cniRes, err := cnidel.DelegateInterfaceSetup(&cniConf,netClientStub,testNet,testEp) if (err != nil && !tc.isErrorExpected) || (err == nil && tc.isErrorExpected) { var detailedErrorMessage string diff --git a/user-guide.md b/user-guide.md index 7f88324c..b8a8dc80 100644 --- a/user-guide.md +++ b/user-guide.md @@ -23,6 +23,7 @@ * [DANM IPVLAN CNI](#danm-ipvlan-cni) * [Device Plugin Support](#device-plugin-support) * [Using Intel SR-IOV CNI](#using-intel-sr-iov-cni) + * [DPDK support](#dpdk-support) * [Usage of DANM's Webhook component](#usage-of-danms-webhook-component) * [Responsibilities](#responsibilities) * [Connecting TenantNetworks to TenantConfigs](#connecting-tenantnetworks-to-tenantconfigs) @@ -259,7 +260,8 @@ The CNI provisions IPVLAN interfaces in L2 mode, and supports the following extr * provisioning generic IP routes into a configured routing table inside the Pod's network namespace * Pod-level controlled provisioning of policy-based IP routes into Pod's network namespace #### Device Plugin support -DANM provides general support to CNIs which interwork with Kubernetes' Device Plugin mechanism such as SR-IOV CNI. +DANM provides general support for CNIs interworking with Kubernetes' Device Plugin mechanism. +A practical example of such a network provisioner is the SR-IOV CNI. When a properly configured Network Device Plugin runs, the allocatable resource list for the node should be updated with resource discovered by the plugin. ##### Using Intel SR-IOV CNI SR-IOV Network Device Plugin allows to create a list of *netdevice* type resource definitions with *sriovMode*, where each resource definition can have one or more assigned *rootDevice* (Physical Function). The plugin looks for Virtual Functions (VF) for each configured Physical Function (PF) and adds all discovered VFs to the allocatable resource's list of the given Kubernetes Node. The Device Plugin resource name will be the device pool name on the Node. These device pools can be referred in Pod definition's resource request part on the usual way. @@ -329,6 +331,17 @@ spec: nodeSelector: sriov: enabled ``` +##### DPDK support +DANM's SR-IOV integration supports -and is tested with- both Intel, and Mellanox manufactured physical functions. +Moreover Pods can use the allocated Virtual Functions for either kernel, or userspace networking. +The only restriction to keep in mind is when a DPDK using application requests VFs from an Intel NIC for the purpose of userspace networking, +those VFs shall be already bound to the vfio-pci kernel driver before the Pod is instantiated. +To guarantee such VFs are always available on the Node the Pod is scheduled to, we strongly suggest advertising vfio-pci bound VFs as a separate Device Pool. +When an already vfio bound function is mounted to an application, DANM also creates a dummy kernel interface in its stead in the Pod's network namespace. +The dummy interface can be easily identified by the application, because it's named exactly as the VF would be. +The dummy interface is used to communicate the IPAM details belonging to the non-kernel managed device, such as IP addresses, IP routes etc. +Userspace applications can interrogate this information via the usual kernel APIs, and then configure the allocated resources into their own IP stack! + ### Usage of DANM's Webhook component #### Responsibilities The Webhook component introduced in DANM V4 is responsible for three things: