Adding support for Intel Fortville + DPDK use-case

crd/apis/danm/v1/types.go

@@ -87,7 +87,7 @@ type DanmEpIface struct {
   MacAddress  string            `json:"MacAddress"`
   Proutes     map[string]string `json:"proutes"`
   Proutes6    map[string]string `json:"proutes6"`
-  DeviceID  string              `json:"DeviceID,omitempty"`
+  DeviceID    string            `json:"DeviceID,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

glide.yaml

@@ -19,6 +19,5 @@ import:
   version: v0.8.1
 - package: github.com/intel/multus-cni
   version: e723aabca8792ddd181167660e252a127a81b073
-- package: github.com/intel/sriov-cni
-  version: 9e4c973b2ac517c64867e33d61aee152d70dc330
-
+- package: github.com/intel/sriov-cni	
+  version: 365c8f8cc1204df84f3e976ea30f113e733ca665

pkg/cnidel/cniconfs.go

@@ -67,13 +67,11 @@ func getSriovCniConfig(netInfo *danmtypes.DanmNet, ipamOptions datastructs.IpamC
   if err != nil {
     return nil, errors.New("failed to get the name of the sriov PF for device "+ ep.Spec.Iface.DeviceID +" due to:" + err.Error())
   }
-  sriovConfig.PfName   = pfname
-  sriovConfig.L2Mode   = true
+  sriovConfig.Master   = pfname
   sriovConfig.Vlan     = netInfo.Spec.Options.Vlan
   sriovConfig.DeviceID = ep.Spec.Iface.DeviceID
   if len(ipamOptions.Ips) > 0 {
     sriovConfig.Ipam   = ipamOptions
-    sriovConfig.L2Mode = false
   }
   rawConfig, err := json.Marshal(sriovConfig)
   if err != nil {

pkg/cnidel/cnidel.go

@@ -75,10 +75,9 @@ func DelegateInterfaceSetup(netConf *datastructs.NetConf, danmClient danmclients
   if cniResult != nil {
     setEpIfaceAddress(cniResult, &ep.Spec.Iface)
   }
-  err = danmep.CreateRoutesInNetNs(*ep, netInfo)
+  err = danmep.PostProcessInterface(*ep, netInfo)
   if err != nil {
-    // We don't consider this serious error, so we only log a warning about the issue.
-    log.Println("WARNING: Could not create IP routes for CNI:" + cniType + " because:" + err.Error())
+    return nil, errors.New("Post-processing failed for interface:" + ep.Spec.Iface.Name + " because:" + err.Error())
   }
   return cniResult, nil
 }

pkg/cnidel/types.go

@@ -2,6 +2,7 @@ package cnidel
 
 import (
   "github.com/containernetworking/cni/pkg/types"
+  sriov_types "github.com/intel/sriov-cni/pkg/types"
   danmtypes "github.com/nokia/danm/crd/apis/danm/v1"
   "github.com/nokia/danm/pkg/datastructs"
 )
@@ -17,24 +18,9 @@ type cniBackendConfig struct {
 
 // sriovNet represent the configuration of sriov cni v1.0.0
 type SriovNet struct {
-  types.NetConf
-  // name of the PF since sriov cni v1.0.0
-  PfName string     `json:"master"`
-  // if true then add VF as L2 mode only, IPAM will not be executed
-  L2Mode bool       `json:"l2enable,omitEmpty"`
-  // VLAN ID to assign for the VF
-  Vlan   int        `json:"vlan,omitEmpty"`
+  sriov_types.NetConf
   // IPAM configuration to be used for this network
   Ipam   datastructs.IpamConfig `json:"ipam,omitEmpty"`
-  // Device PCI ID
-  DeviceID string `json:"deviceID"`
-}
-
-// VfInformation is a DeviceInfo descriptor expected by sriov cni v1.0.0
-type VfInformation struct {
-  PCIaddr string `json:"pci_addr"`
-  Pfname  string `json:"pfname"`
-  Vfid    int    `json:"vfid"`
 }
 
 type MacvlanNet struct {

pkg/danmep/danmep.go

@@ -10,6 +10,7 @@ import (
   "github.com/containernetworking/plugins/pkg/ns"
   "github.com/containernetworking/plugins/pkg/utils/sysctl"
   meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+  sriov_utils "github.com/intel/sriov-cni/pkg/utils"
   danmtypes "github.com/nokia/danm/crd/apis/danm/v1"
   danmclientset "github.com/nokia/danm/crd/client/clientset/versioned"
 )
@@ -107,7 +108,12 @@ func DetermineHostDeviceName(dnet *danmtypes.DanmNet) string {
   return device
 }
 
-func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
+func PostProcessInterface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
+  if ep.Spec.Iface.DeviceID == "" &&
+     len(ep.Spec.Iface.Proutes)    == 0 && len(ep.Spec.Iface.Proutes6)    == 0 &&
+     len(dnet.Spec.Options.Routes) == 0 && len(dnet.Spec.Options.Routes6) == 0 {
+    return nil
+  }
   runtime.LockOSThread()
   defer runtime.UnlockOSThread()
   origNs, err := ns.GetCurrentNS()
@@ -127,7 +133,14 @@ func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
   }()
   err = hns.Set()
   if err != nil {
-    return errors.New("failed to enter network namespace of CID:"+ep.Spec.Netns+" with error:"+err.Error())
+    return errors.New("failed to enter network namespace of CID:" + ep.Spec.Netns + " with error:" + err.Error())
+  }
+  isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID)
+  if isVfAttachedToDpdkDriver {
+    err = createDummyInterface(ep)
+    if err != nil {
+      return err
+    }
   }
   return addIpRoutes(ep,dnet)
 }
@@ -176,7 +189,8 @@ func isIPv6Needed(ep danmtypes.DanmEp) bool {
 }
 
 func isIPv6NotNeeded(ep danmtypes.DanmEp) bool {
-  if ep.Spec.Iface.AddressIPv6 == "" {
+  isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID)
+  if !isVfAttachedToDpdkDriver && ep.Spec.Iface.AddressIPv6 == "" {
     return true
   }
   return false

pkg/danmep/ep.go

@@ -29,12 +29,7 @@ func createIpvlanInterface(dnet *danmtypes.DanmNet, ep danmtypes.DanmEp) error {
   return createContainerIface(ep, dnet, device)
 }
 
-// TODO: Refactor this, as cyclomatic complexity is too high
 func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device string) error {
-  var (
-    addr4, addr6 net.IP
-    pref4, pref6 *net.IPNet
-  )
   runtime.LockOSThread()
   defer runtime.UnlockOSThread()
   origns, err := ns.GetCurrentNS()
@@ -87,49 +82,58 @@ func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device s
   if err != nil {
     return errors.New("cannot find IPVLAN interface in network namespace:" + err.Error())
   }
-  ip := ep.Spec.Iface.Address
-  if ip != "" {
-    addr4, pref4, err = net.ParseCIDR(ip)
-    if err != nil {
-      return errors.New("cannot parse ip4 address because:" + err.Error())
-    }
-    ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr4, Mask: pref4.Mask}}
-    err = netlink.AddrAdd(iface, ipAddr)
+  err = configureLink(iface, ep)
+  if err != nil {
+    return err
+  }
+  if ep.Spec.Iface.Address != "" {
+    addr,_,_ := net.ParseCIDR(ep.Spec.Iface.Address)
+    err = arping.GratuitousArpOverIfaceByName(addr, ep.Spec.Iface.Name)
     if err != nil {
-      return errors.New("Cannot add ip4 address to IPVLAN interface because:" + err.Error())
+      log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!")
     }
   }
-  ip6 := ep.Spec.Iface.AddressIPv6
-  // TODO: Refactor, duplicate of 87-98
-  if ip6 != "" {
-    addr6, pref6, err = net.ParseCIDR(ip6)
+  err = addIpRoutes(ep, dnet)
+  if err != nil {
+    return errors.New("IP routes could not be provisioned, because:" + err.Error())
+  }
+  return nil
+}
+
+func configureLink(iface netlink.Link, ep danmtypes.DanmEp) error {
+  var err error
+  if ep.Spec.Iface.Address != "" {
+    err = addIpToLink(ep.Spec.Iface.Address, iface)
     if err != nil {
-      return errors.New("cannot parse ip6 address because:" + err.Error())
+      return err
     }
-    ipAddr6 := &netlink.Addr{IPNet: &net.IPNet{IP: addr6, Mask: pref6.Mask}}
-    err = netlink.AddrAdd(iface, ipAddr6)
+  }
+  if ep.Spec.Iface.AddressIPv6 != "" {
+    err = addIpToLink(ep.Spec.Iface.AddressIPv6, iface)
     if err != nil {
-      return errors.New("Cannot add ip6 address to IPVLAN interface because:" + err.Error())
+      return err
     }
   }
-  dstPrefix := ep.Spec.Iface.Name
-  err = netlink.LinkSetName(iface, dstPrefix)
+  err = netlink.LinkSetName(iface, ep.Spec.Iface.Name)
   if err != nil {
-    return errors.New("cannot rename IPVLAN interface because:" + err.Error())
+    return errors.New("cannot rename link:" + ep.Spec.Iface.Name + " because:" + err.Error())
   }
   err = netlink.LinkSetUp(iface)
   if err != nil {
-    return errors.New("cannot set renamed IPVLAN interface to up because:" + err.Error())
+    return errors.New("cannot set link:" + ep.Spec.Iface.Name + " UP because:" + err.Error())
   }
-  if ip != "" {
-    err = arping.GratuitousArpOverIfaceByName(addr4, dstPrefix)
-    if err != nil {
-      log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!")
-    }
+  return nil
+}
+
+func addIpToLink(ip string, iface netlink.Link) error {
+  addr, pref, err := net.ParseCIDR(ip)
+  if err != nil {
+    return errors.New("cannot parse IP address because:" + err.Error())
   }
-  err = addIpRoutes(ep, dnet)
+  ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr, Mask: pref.Mask}}
+  err = netlink.AddrAdd(iface, ipAddr)
   if err != nil {
-    return errors.New("IP routes could not be provisioned, because:" + err.Error())
+    return errors.New("cannot add IP address to link because:" + err.Error())
   }
   return nil
 }
@@ -212,7 +216,7 @@ func deleteContainerIface(ep danmtypes.DanmEp) error {
   defer runtime.UnlockOSThread()
   origns, err := ns.GetCurrentNS()
   if err != nil {
-    return errors.New("getting the current netNS failed")  
+    return errors.New("getting the current netNS failed")
   }
   hns, err := ns.GetNS(ep.Spec.Netns)
   if err != nil {
@@ -259,3 +263,20 @@ func deleteEp(ep danmtypes.DanmEp) error {
   }
   return deleteContainerIface(ep)
 }
+
+func createDummyInterface(ep danmtypes.DanmEp) error {
+  dummy := &netlink.Dummy {
+    LinkAttrs: netlink.LinkAttrs {
+      Name: ep.Spec.Iface.Name,
+    },
+  }
+  err := netlink.LinkAdd(dummy)
+  if err != nil {
+    return errors.New("cannot create dummy interface for DPDK because:" + err.Error())
+  }
+  iface, err := netlink.LinkByName(ep.Spec.Iface.Name)
+  if err != nil {
+    return errors.New("cannot find freshly created dummy interface because:" + err.Error())
+  }
+  return configureLink(iface, ep)
+}