Adding support for Intel Fortville + DPDK use-case

crd/apis/danm/v1/types.go

@@ -87,7 +87,7 @@ type DanmEpIface struct {
   MacAddress  string            `json:"MacAddress"`
   Proutes     map[string]string `json:"proutes"`
   Proutes6    map[string]string `json:"proutes6"`
-  DeviceID  string              `json:"DeviceID,omitempty"`
+  DeviceID    string            `json:"DeviceID,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

glide.yaml

@@ -19,6 +19,5 @@ import:
   version: v0.8.1
 - package: github.com/intel/multus-cni
   version: e723aabca8792ddd181167660e252a127a81b073
-- package: github.com/intel/sriov-cni
-  version: 9e4c973b2ac517c64867e33d61aee152d70dc330
-
+- package: github.com/intel/sriov-cni	
+  version: 365c8f8cc1204df84f3e976ea30f113e733ca665

pkg/cnidel/cniconfs.go

@@ -62,18 +62,15 @@ func getSriovCniConfig(netInfo *danmtypes.DanmNet, ipamOptions datastructs.IpamC
   sriovConfig.CNIVersion = cniVersion
   sriovConfig.Name       = netInfo.Spec.NetworkID
   sriovConfig.Type       = "sriov"
-  // initialize SriovNet specific fields:
   pfname, err := sriov_utils.GetPfName(ep.Spec.Iface.DeviceID)
   if err != nil {
     return nil, errors.New("failed to get the name of the sriov PF for device "+ ep.Spec.Iface.DeviceID +" due to:" + err.Error())
   }
-  sriovConfig.PfName   = pfname
-  sriovConfig.L2Mode   = true
+  sriovConfig.Master   = pfname
   sriovConfig.Vlan     = netInfo.Spec.Options.Vlan
   sriovConfig.DeviceID = ep.Spec.Iface.DeviceID
   if len(ipamOptions.Ips) > 0 {
     sriovConfig.Ipam   = ipamOptions
-    sriovConfig.L2Mode = false
   }
   rawConfig, err := json.Marshal(sriovConfig)
   if err != nil {

pkg/cnidel/cnidel.go

@@ -13,7 +13,6 @@ import (
   "github.com/containernetworking/cni/pkg/types"
   "github.com/containernetworking/cni/pkg/types/current"
   "github.com/containernetworking/cni/pkg/version"
-  "github.com/nokia/danm/pkg/danmep"
   "github.com/nokia/danm/pkg/datastructs"
   "github.com/nokia/danm/pkg/ipam"
   "github.com/nokia/danm/pkg/netcontrol"
@@ -58,28 +57,23 @@ func DelegateInterfaceSetup(netConf *datastructs.NetConf, danmClient danmclients
     //As netInfo is only copied to IPAM above, the IP allocation is not refreshed in the original copy.
     //Without re-reading the network body we risk leaking IPs if error happens later on within the same thread!
     netInfo,err = netcontrol.GetNetworkFromEp(danmClient, *ep)
-    if err != nil {log.Println("lofasz:" + err.Error())}
+    if err != nil {
+      return nil, err
+    }
     ipamOptions = getCniIpamConfig(netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
   }
   rawConfig, err := getCniPluginConfig(netConf, netInfo, ipamOptions, ep)
   if err != nil {
-    FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
     return nil, err
   }
   cniType := netInfo.Spec.NetworkType
   cniResult,err := execCniPlugin(cniType, CniAddOp, netInfo, rawConfig, ep)
   if err != nil {
-    FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
     return nil, errors.New("Error delegating ADD to CNI plugin:" + cniType + " because:" + err.Error())
   }
   if cniResult != nil {
     setEpIfaceAddress(cniResult, &ep.Spec.Iface)
   }
-  err = danmep.CreateRoutesInNetNs(*ep, netInfo)
-  if err != nil {
-    // We don't consider this serious error, so we only log a warning about the issue.
-    log.Println("WARNING: Could not create IP routes for CNI:" + cniType + " because:" + err.Error())
-  }
   return cniResult, nil
 }
 

pkg/cnidel/types.go

@@ -2,6 +2,7 @@ package cnidel
 
 import (
   "github.com/containernetworking/cni/pkg/types"
+  sriov_types "github.com/intel/sriov-cni/pkg/types"
   danmtypes "github.com/nokia/danm/crd/apis/danm/v1"
   "github.com/nokia/danm/pkg/datastructs"
 )
@@ -17,24 +18,9 @@ type cniBackendConfig struct {
 
 // sriovNet represent the configuration of sriov cni v1.0.0
 type SriovNet struct {
-  types.NetConf
-  // name of the PF since sriov cni v1.0.0
-  PfName string     `json:"master"`
-  // if true then add VF as L2 mode only, IPAM will not be executed
-  L2Mode bool       `json:"l2enable,omitEmpty"`
-  // VLAN ID to assign for the VF
-  Vlan   int        `json:"vlan,omitEmpty"`
+  sriov_types.NetConf
   // IPAM configuration to be used for this network
   Ipam   datastructs.IpamConfig `json:"ipam,omitEmpty"`
-  // Device PCI ID
-  DeviceID string `json:"deviceID"`
-}
-
-// VfInformation is a DeviceInfo descriptor expected by sriov cni v1.0.0
-type VfInformation struct {
-  PCIaddr string `json:"pci_addr"`
-  Pfname  string `json:"pfname"`
-  Vfid    int    `json:"vfid"`
 }
 
 type MacvlanNet struct {

pkg/danmep/danmep.go

@@ -10,6 +10,7 @@ import (
   "github.com/containernetworking/plugins/pkg/ns"
   "github.com/containernetworking/plugins/pkg/utils/sysctl"
   meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+  sriov_utils "github.com/intel/sriov-cni/pkg/utils"
   danmtypes "github.com/nokia/danm/crd/apis/danm/v1"
   danmclientset "github.com/nokia/danm/crd/client/clientset/versioned"
 )
@@ -107,7 +108,7 @@ func DetermineHostDeviceName(dnet *danmtypes.DanmNet) string {
   return device
 }
 
-func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
+func PostProcessInterface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
   runtime.LockOSThread()
   defer runtime.UnlockOSThread()
   origNs, err := ns.GetCurrentNS()
@@ -127,33 +128,24 @@ func CreateRoutesInNetNs(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet) error {
   }()
   err = hns.Set()
   if err != nil {
-    return errors.New("failed to enter network namespace of CID:"+ep.Spec.Netns+" with error:"+err.Error())
+    return errors.New("failed to enter network namespace of CID:" + ep.Spec.Netns + " with error:" + err.Error())
   }
-  return addIpRoutes(ep,dnet)
-}
-
-func SetDanmEpSysctls(ep danmtypes.DanmEp) error {
-  runtime.LockOSThread()
-  defer runtime.UnlockOSThread()
-  // save the current namespace
-  origNs, err := ns.GetCurrentNS()
-  if err != nil {
-    return errors.New("failed to get current network namespace due to:" + err.Error())
-  }
-  // enter to the Pod's network namespace
-  podNs, err := ns.GetNS(ep.Spec.Netns)
-  if err != nil {
-    return errors.New("failed to get Pod's network namespace due to:" + err.Error())
+  isVfAttachedToDpdkDriver,_ := sriov_utils.HasDpdkDriver(ep.Spec.Iface.DeviceID)
+  if isVfAttachedToDpdkDriver {
+    err = createDummyInterface(ep)
+    if err != nil {
+      return errors.New("failed to create dummy kernel interface for " + ep.Spec.Iface.Name + " because:" + err.Error())
+    }
   }
-  err = podNs.Set()
+  err = setDanmEpSysctls(ep)
   if err != nil {
-    return errors.New("failed to switch to Pod's network namespace due to:" + err.Error())
+    return errors.New("failed to set kernel configs for interface" + ep.Spec.Iface.Name + " beause:" + err.Error())
   }
-  defer func() {
-    podNs.Close()
-    origNs.Set()
-  }()
-  // set sysctls
+  return addIpRoutes(ep,dnet)
+}
+
+func setDanmEpSysctls(ep danmtypes.DanmEp) error {
+  var err error
   for _, s := range sysctls {
     if s.sysctlFunc(ep) {
       for _, ss := range s.sysctlData {

pkg/danmep/ep.go

@@ -29,12 +29,7 @@ func createIpvlanInterface(dnet *danmtypes.DanmNet, ep danmtypes.DanmEp) error {
   return createContainerIface(ep, dnet, device)
 }
 
-// TODO: Refactor this, as cyclomatic complexity is too high
 func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device string) error {
-  var (
-    addr4, addr6 net.IP
-    pref4, pref6 *net.IPNet
-  )
   runtime.LockOSThread()
   defer runtime.UnlockOSThread()
   origns, err := ns.GetCurrentNS()
@@ -87,49 +82,54 @@ func createContainerIface(ep danmtypes.DanmEp, dnet *danmtypes.DanmNet, device s
   if err != nil {
     return errors.New("cannot find IPVLAN interface in network namespace:" + err.Error())
   }
-  ip := ep.Spec.Iface.Address
-  if ip != "" {
-    addr4, pref4, err = net.ParseCIDR(ip)
-    if err != nil {
-      return errors.New("cannot parse ip4 address because:" + err.Error())
-    }
-    ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr4, Mask: pref4.Mask}}
-    err = netlink.AddrAdd(iface, ipAddr)
+  err = configureLink(iface, ep)
+  if err != nil {
+    return err
+  }
+  if ep.Spec.Iface.Address != "" {
+    addr,_,_ := net.ParseCIDR(ep.Spec.Iface.Address)
+    err = arping.GratuitousArpOverIfaceByName(addr, ep.Spec.Iface.Name)
     if err != nil {
-      return errors.New("Cannot add ip4 address to IPVLAN interface because:" + err.Error())
+      log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!")
     }
   }
-  ip6 := ep.Spec.Iface.AddressIPv6
-  // TODO: Refactor, duplicate of 87-98
-  if ip6 != "" {
-    addr6, pref6, err = net.ParseCIDR(ip6)
+  return nil
+}
+
+func configureLink(iface netlink.Link, ep danmtypes.DanmEp) error {
+  var err error
+  if ep.Spec.Iface.Address != "" {
+    err = addIpToLink(ep.Spec.Iface.Address, iface)
     if err != nil {
-      return errors.New("cannot parse ip6 address because:" + err.Error())
+      return err
     }
-    ipAddr6 := &netlink.Addr{IPNet: &net.IPNet{IP: addr6, Mask: pref6.Mask}}
-    err = netlink.AddrAdd(iface, ipAddr6)
+  }
+  if ep.Spec.Iface.AddressIPv6 != "" {
+    err = addIpToLink(ep.Spec.Iface.AddressIPv6, iface)
     if err != nil {
-      return errors.New("Cannot add ip6 address to IPVLAN interface because:" + err.Error())
+      return err
     }
   }
-  dstPrefix := ep.Spec.Iface.Name
-  err = netlink.LinkSetName(iface, dstPrefix)
+  err = netlink.LinkSetName(iface, ep.Spec.Iface.Name)
   if err != nil {
-    return errors.New("cannot rename IPVLAN interface because:" + err.Error())
+    return errors.New("cannot rename link:" + ep.Spec.Iface.Name + " because:" + err.Error())
   }
   err = netlink.LinkSetUp(iface)
   if err != nil {
-    return errors.New("cannot set renamed IPVLAN interface to up because:" + err.Error())
+    return errors.New("cannot set link:" + ep.Spec.Iface.Name + " UP because:" + err.Error())
   }
-  if ip != "" {
-    err = arping.GratuitousArpOverIfaceByName(addr4, dstPrefix)
-    if err != nil {
-      log.Println("WARNING: sending gARP failed with error:" + err.Error(), ", but we will ignore that for now!")
-    }
+  return nil
+}
+
+func addIpToLink(ip string, iface netlink.Link) error {
+  addr, pref, err := net.ParseCIDR(ip)
+  if err != nil {
+    return errors.New("cannot parse IP address because:" + err.Error())
   }
-  err = addIpRoutes(ep, dnet)
+  ipAddr := &netlink.Addr{IPNet: &net.IPNet{IP: addr, Mask: pref.Mask}}
+  err = netlink.AddrAdd(iface, ipAddr)
   if err != nil {
-    return errors.New("IP routes could not be provisioned, because:" + err.Error())
+    return errors.New("cannot add IP address to link because:" + err.Error())
   }
   return nil
 }
@@ -212,7 +212,7 @@ func deleteContainerIface(ep danmtypes.DanmEp) error {
   defer runtime.UnlockOSThread()
   origns, err := ns.GetCurrentNS()
   if err != nil {
-    return errors.New("getting the current netNS failed")  
+    return errors.New("getting the current netNS failed")
   }
   hns, err := ns.GetNS(ep.Spec.Netns)
   if err != nil {
@@ -259,3 +259,20 @@ func deleteEp(ep danmtypes.DanmEp) error {
   }
   return deleteContainerIface(ep)
 }
+
+func createDummyInterface(ep danmtypes.DanmEp) error {
+  dummy := &netlink.Dummy {
+    LinkAttrs: netlink.LinkAttrs {
+      Name: ep.Spec.Iface.Name,
+    },
+  }
+  err := netlink.LinkAdd(dummy)
+  if err != nil {
+    return errors.New("cannot create dummy interface for DPDK because:" + err.Error())
+  }
+  iface, err := netlink.LinkByName(ep.Spec.Iface.Name)
+  if err != nil {
+    return errors.New("cannot find freshly created dummy interface because:" + err.Error())
+  }
+  return configureLink(iface, ep)
+}

pkg/metacni/metacni.go

@@ -369,19 +369,20 @@ func createDelegatedInterface(syncher *syncher.Syncher, danmClient danmclientset
   }
   delegatedResult,err := cnidel.DelegateInterfaceSetup(DanmConfig, danmClient, netInfo, &ep)
   if err != nil {
+    cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
     syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("CNI delegation failed due to error:" + err.Error()), nil)
     return
   }
-  err = danmep.PutDanmEp(danmClient, ep)
+  err = danmep.PostProcessInterface(ep, netInfo)
   if err != nil {
     cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
-    syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("DanmEp object could not be PUT to K8s due to error:" + err.Error()), nil)
+    syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("Post-processing failed for interface:" + ep.Spec.Iface.Name + " because:" + err.Error()), nil)
     return
   }
-  err = danmep.SetDanmEpSysctls(ep)
+  err = danmep.PutDanmEp(danmClient, ep)
   if err != nil {
     cnidel.FreeDelegatedIps(danmClient, netInfo, ep.Spec.Iface.Address, ep.Spec.Iface.AddressIPv6)
-    syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("Sysctls could not be set due to error:" + err.Error()), nil)
+    syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("DanmEp object could not be PUT to K8s due to error:" + err.Error()), nil)
     return
   }
   syncher.PushResult(netInfo.ObjectMeta.Name, nil, delegatedResult)
@@ -423,7 +424,7 @@ func createDanmInterface(syncher *syncher.Syncher, danmClient danmclientset.Inte
     syncher.PushResult(netInfo.ObjectMeta.Name, errors.New("IPVLAN interface could not be created due to error:" + err.Error()), nil)
     return
   }
-  err = danmep.SetDanmEpSysctls(ep)
+  err = danmep.PostProcessInterface(ep, netInfo)
   if err != nil {
     ipam.GarbageCollectIps(danmClient, netInfo, ip4, ip6)
     danmClient.DanmV1().DanmEps(ep.ObjectMeta.Namespace).Delete(ep.ObjectMeta.Name, &meta_v1.DeleteOptions{})