From e3d661848000630b444733b3ac01978730dd4fc8 Mon Sep 17 00:00:00 2001 From: Levente Kale Date: Fri, 27 Sep 2019 17:26:38 +0200 Subject: [PATCH] Adding support for Intel Fortville + DPDK use-case --- crd/apis/danm/v1/types.go | 2 +- glide.lock | 7 +-- glide.yaml | 5 +-- pkg/cnidel/cniconfs.go | 4 +- pkg/cnidel/cnidel.go | 5 +-- pkg/cnidel/types.go | 18 +------- pkg/danmep/danmep.go | 20 +++++++-- pkg/danmep/ep.go | 89 ++++++++++++++++++++++++--------------- 8 files changed, 84 insertions(+), 66 deletions(-) diff --git a/crd/apis/danm/v1/types.go b/crd/apis/danm/v1/types.go index 870dfc66..5954db10 100644 --- a/crd/apis/danm/v1/types.go +++ b/crd/apis/danm/v1/types.go @@ -87,7 +87,7 @@ type DanmEpIface struct { MacAddress string `json:"MacAddress"` Proutes map[string]string `json:"proutes"` Proutes6 map[string]string `json:"proutes6"` - DeviceID string `json:"DeviceID,omitempty"` + DeviceID string `json:"DeviceID,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/glide.lock b/glide.lock index b0a5fe53..253c6e24 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 1d7eea7c8fe94fb460fead10727227389ddead184b862425f9a6a913ec4b6e94 -updated: 2019-08-06T22:31:42.529643558+02:00 +hash: aa386d47de4210547ee89573057b83178ae6f7f4b1dd1df33f0cde11cf69b228 +updated: 2019-09-27T17:53:22.774886331+02:00 imports: - name: github.com/apparentlymart/go-cidr version: 2bd8b58cf4275aeb086ade613de226773e29e853 @@ -72,8 +72,9 @@ imports: - logging - types - name: github.com/intel/sriov-cni - version: 9e4c973b2ac517c64867e33d61aee152d70dc330 + version: 365c8f8cc1204df84f3e976ea30f113e733ca665 subpackages: + - pkg/types - pkg/utils - name: github.com/j-keck/arping version: 2cf9dc699c5640a7e2c81403a44127bf28033600 diff --git a/glide.yaml b/glide.yaml index e11bbfa8..470cdead 100644 --- a/glide.yaml +++ b/glide.yaml @@ -19,6 +19,5 @@ import: version: v0.8.1 - package: github.com/intel/multus-cni version: e723aabca8792ddd181167660e252a127a81b073 -- package: github.com/intel/sriov-cni - version: 9e4c973b2ac517c64867e33d61aee152d70dc330 - +- package: github.com/intel/sriov-cni + version: 365c8f8cc1204df84f3e976ea30f113e733ca665 diff --git a/pkg/cnidel/cniconfs.go b/pkg/cnidel/cniconfs.go index 4c761417..c41790c7 100644 --- a/pkg/cnidel/cniconfs.go +++ b/pkg/cnidel/cniconfs.go @@ -67,13 +67,11 @@ func getSriovCniConfig(netInfo *danmtypes.DanmNet, ipamOptions datastructs.IpamC if err != nil { return nil, errors.New("failed to get the name of the sriov PF for device "+ ep.Spec.Iface.DeviceID +" due to:" + err.Error()) } - sriovConfig.PfName = pfname - sriovConfig.L2Mode = true + sriovConfig.Master = pfname sriovConfig.Vlan = netInfo.Spec.Options.Vlan sriovConfig.DeviceID = ep.Spec.Iface.DeviceID if len(ipamOptions.Ips) > 0 { sriovConfig.Ipam = ipamOptions - sriovConfig.L2Mode = false } rawConfig, err := json.Marshal(sriovConfig) if err != nil { diff --git a/pkg/cnidel/cnidel.go b/pkg/cnidel/cnidel.go index 03eb9eab..69ec2b0f 100644 --- a/pkg/cnidel/cnidel.go +++ b/pkg/cnidel/cnidel.go @@ -75,10 +75,9 @@ func DelegateInterfaceSetup(netConf *datastructs.NetConf, danmClient danmclients if cniResult != nil { setEpIfaceAddress(cniResult, &ep.Spec.Iface) } - err = danmep.CreateRoutesInNetNs(*ep, netInfo) + err = danmep.PostProcessInterface(*ep, netInfo) if err != nil { - // We don't consider this serious error, so we only log a warning about the issue. - log.Println("WARNING: Could not create IP routes for CNI:" + cniType + " because:" + err.Error()) + return nil, errors.New("Post-processing failed for interface:" + ep.Spec.Iface.Name + " because:" + err.Error()) } return cniResult, nil } diff --git a/pkg/cnidel/types.go b/pkg/cnidel/types.go index a4db42d5..e820b19e 100644 --- a/pkg/cnidel/types.go +++ b/pkg/cnidel/types.go @@ -2,6 +2,7 @@ package cnidel import ( "github.com/containernetworking/cni/pkg/types" + sriov_types "github.com/intel/sriov-cni/pkg/types" danmtypes "github.com/nokia/danm/crd/apis/danm/v1" "github.com/nokia/danm/pkg/datastructs" ) @@ -17,24 +18,9 @@ type cniBackendConfig struct { // sriovNet represent the configuration of sriov cni v1.0.0 type SriovNet struct { - types.NetConf - // name of the PF since sriov cni v1.0.0 - PfName string `json:"master"` - // if true then add VF as L2 mode only, IPAM will not be executed - L2Mode bool `json:"l2enable,omitEmpty"` - // VLAN ID to assign for the VF - Vlan int `json:"vlan,omitEmpty"` + sriov_types.NetConf // IPAM configuration to be used for this network Ipam datastructs.IpamConfig `json:"ipam,omitEmpty"` - // Device PCI ID - DeviceID string `json:"deviceID"` -} - -// VfInformation is a DeviceInfo descriptor expected by sriov cni v1.0.0 -type VfInformation struct { - PCIaddr string `json:"pci_addr"` - Pfname string `json:"pfname"` - Vfid int `json:"vfid"` } type MacvlanNet struct { diff --git a/pkg/danmep/danmep.go b/pkg/danmep/danmep.go index 52bf8669..f8636586 100644 --- a/pkg/danmep/danmep.go +++ b/pkg/danmep/danmep.go @@ -10,6 +10,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils/sysctl" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + sriov_utils "github.com/intel/sriov-cni/pkg/utils" danmtypes "github.com/nokia/danm/crd/apis/danm/v1" danmclientset "github.com/nokia/danm/crd/client/clientset/versioned" ) @@ -107,7 +108,12 @@ func DetermineHostDeviceName(dnet *danmtypes.DanmNet) string { return device } -func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { +func PostProcessInterface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { + if ep.Spec.Iface.DeviceID == "" && + len(ep.Spec.Iface.Proutes) == 0 && len(ep.Spec.Iface.Proutes6) == 0 && + len(dnet.Spec.Options.Routes) == 0 && len(dnet.Spec.Options.Routes6) == 0 { + return nil + } runtime.LockOSThread() defer runtime.UnlockOSThread() origNs, err := ns.GetCurrentNS() @@ -127,7 +133,14 @@ func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error { }() err = hns.Set() if err != nil { - return errors.New("failed to enter network namespace of CID:"+ep.Spec.Netns+" with error:"+err.Error()) + return errors.New("failed to enter network namespace of CID:" + ep.Spec.Netns + " with error:" + err.Error()) + } + isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID) + if isVfAttachedToDpdkDriver { + err = createDummyInterface(ep) + if err != nil { + return err + } } return addIpRoutes(ep,dnet) } @@ -176,7 +189,8 @@ func isIPv6Needed(ep danmtypes.DanmEp) bool { } func isIPv6NotNeeded(ep danmtypes.DanmEp) bool { - if ep.Spec.Iface.AddressIPv6 == "" { + isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID) + if !isVfAttachedToDpdkDriver && ep.Spec.Iface.AddressIPv6 == "" { return true } return false diff --git a/pkg/danmep/ep.go b/pkg/danmep/ep.go index c70b6239..75fb92a0 100644 --- a/pkg/danmep/ep.go +++ b/pkg/danmep/ep.go @@ -29,12 +29,7 @@ func createIpvlanInterface(dnet *danmtypes.DanmNet, ep danmtypes.DanmEp) error { return createContainerIface(ep, dnet, device) } -// TODO: Refactor this, as cyclomatic complexity is too high func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device string) error { - var ( - addr4, addr6 net.IP - pref4, pref6 *net.IPNet - ) runtime.LockOSThread() defer runtime.UnlockOSThread() origns, err := ns.GetCurrentNS() @@ -87,49 +82,58 @@ func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device s if err != nil { return errors.New("cannot find IPVLAN interface in network namespace:" + err.Error()) } - ip := ep.Spec.Iface.Address - if ip != "" { - addr4, pref4, err = net.ParseCIDR(ip) - if err != nil { - return errors.New("cannot parse ip4 address because:" + err.Error()) - } - ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr4, Mask: pref4.Mask}} - err = netlink.AddrAdd(iface, ipAddr) + err = configureLink(iface, ep) + if err != nil { + return err + } + if ep.Spec.Iface.Address != "" { + addr,_,_ := net.ParseCIDR(ep.Spec.Iface.Address) + err = arping.GratuitousArpOverIfaceByName(addr, ep.Spec.Iface.Name) if err != nil { - return errors.New("Cannot add ip4 address to IPVLAN interface because:" + err.Error()) + log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!") } } - ip6 := ep.Spec.Iface.AddressIPv6 - // TODO: Refactor, duplicate of 87-98 - if ip6 != "" { - addr6, pref6, err = net.ParseCIDR(ip6) + err = addIpRoutes(ep, dnet) + if err != nil { + return errors.New("IP routes could not be provisioned, because:" + err.Error()) + } + return nil +} + +func configureLink(iface netlink.Link, ep danmtypes.DanmEp) error { + var err error + if ep.Spec.Iface.Address != "" { + err = addIpToLink(ep.Spec.Iface.Address, iface) if err != nil { - return errors.New("cannot parse ip6 address because:" + err.Error()) + return err } - ipAddr6 := &netlink.Addr{IPNet: &net.IPNet{IP: addr6, Mask: pref6.Mask}} - err = netlink.AddrAdd(iface, ipAddr6) + } + if ep.Spec.Iface.AddressIPv6 != "" { + err = addIpToLink(ep.Spec.Iface.AddressIPv6, iface) if err != nil { - return errors.New("Cannot add ip6 address to IPVLAN interface because:" + err.Error()) + return err } } - dstPrefix := ep.Spec.Iface.Name - err = netlink.LinkSetName(iface, dstPrefix) + err = netlink.LinkSetName(iface, ep.Spec.Iface.Name) if err != nil { - return errors.New("cannot rename IPVLAN interface because:" + err.Error()) + return errors.New("cannot rename link:" + ep.Spec.Iface.Name + " because:" + err.Error()) } err = netlink.LinkSetUp(iface) if err != nil { - return errors.New("cannot set renamed IPVLAN interface to up because:" + err.Error()) + return errors.New("cannot set link:" + ep.Spec.Iface.Name + " UP because:" + err.Error()) } - if ip != "" { - err = arping.GratuitousArpOverIfaceByName(addr4, dstPrefix) - if err != nil { - log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!") - } + return nil +} + +func addIpToLink(ip string, iface netlink.Link) error { + addr, pref, err := net.ParseCIDR(ip) + if err != nil { + return errors.New("cannot parse IP address because:" + err.Error()) } - err = addIpRoutes(ep, dnet) + ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr, Mask: pref.Mask}} + err = netlink.AddrAdd(iface, ipAddr) if err != nil { - return errors.New("IP routes could not be provisioned, because:" + err.Error()) + return errors.New("cannot add IP address to link because:" + err.Error()) } return nil } @@ -212,7 +216,7 @@ func deleteContainerIface(ep danmtypes.DanmEp) error { defer runtime.UnlockOSThread() origns, err := ns.GetCurrentNS() if err != nil { - return errors.New("getting the current netNS failed") + return errors.New("getting the current netNS failed") } hns, err := ns.GetNS(ep.Spec.Netns) if err != nil { @@ -259,3 +263,20 @@ func deleteEp(ep danmtypes.DanmEp) error { } return deleteContainerIface(ep) } + +func createDummyInterface(ep danmtypes.DanmEp) error { + dummy := &netlink.Dummy { + LinkAttrs: netlink.LinkAttrs { + Name: ep.Spec.Iface.Name, + }, + } + err := netlink.LinkAdd(dummy) + if err != nil { + return errors.New("cannot create dummy interface for DPDK because:" + err.Error()) + } + iface, err := netlink.LinkByName(ep.Spec.Iface.Name) + if err != nil { + return errors.New("cannot find freshly created dummy interface because:" + err.Error()) + } + return configureLink(iface, ep) +} \ No newline at end of file