vpp-manager: Add uplink annotations, default disable qdisc bypass

This adds an annotation map on the uplinkConfiguration object in the json definition allowing to change v2/v3 version for af_packet and the use of qdisc bypass or not. We now default to qdisc bypass disables as this seems to be related to issues on EC2. But this is now exposed in the configuration. Signed-off-by: Nathan Skrzypczak <[email protected]>
projectcalico · Dec 20, 2022 · 82c88a1 · 82c88a1
1 parent d8288e1
commit 82c88a1
Show file tree

Hide file tree

Showing 13 changed files with 43 additions and 16 deletions.
diff --git a/config/config/config.go b/config/config/config.go
@@ -43,7 +43,8 @@ type InterfaceSpec struct {
 	TxQueueSize int   `json:"txqsz"`
 	IsL3        *bool `json:"isl3"`
 	/* "interrupt" "adaptive" or "polling" mode */
-	RxMode types.RxMode `json:"rxMode"`
+	RxMode      types.RxMode      `json:"rxMode"`
+	Annotations map[string]string `json:"annotations"`
 }
 
 func (i *InterfaceSpec) GetIsL3(isMemif bool) bool {

diff --git a/vpp-manager/uplink/af_packet.go b/vpp-manager/uplink/af_packet.go
@@ -16,13 +16,17 @@
 package uplink
 
 import (
+	"strconv"
+
 	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/vishvananda/netlink"
+
 	"github.com/projectcalico/vpp-dataplane/config/config"
 	"github.com/projectcalico/vpp-dataplane/vpp-manager/utils"
 	"github.com/projectcalico/vpp-dataplane/vpplink"
+	"github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/af_packet"
 	"github.com/projectcalico/vpp-dataplane/vpplink/types"
-	log "github.com/sirupsen/logrus"
-	"github.com/vishvananda/netlink"
 )
 
 type AFPacketDriver struct {
@@ -75,14 +79,34 @@ func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *AFPacketDriver) fetchBooleanAnnotation(annotation string, defaultValue bool, uplinkSpec *config.UplinkInterfaceSpec) bool {
+	spec, found := uplinkSpec.Annotations[annotation]
+	if !found {
+		return defaultValue
+	}
+	b, err := strconv.ParseBool(spec)
+	if err != nil {
+		log.WithError(err).Errorf("Error parsing annotation %s '%s'", annotation, spec)
+		return defaultValue
+	}
+	return b
+}
+
+func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid)
 	if err != nil {
 		return errors.Wrap(err, "Moving uplink in NS failed")
 	}
 
-	intf := types.AfPacketInterface{
-		GenericVppInterface: d.getGenericVppInterface(),
+	intf := types.AfPacketInterface{GenericVppInterface: d.getGenericVppInterface()}
+	if d.params.EnableGSO {
+		intf.Flags |= af_packet.AF_PACKET_API_FLAG_CKSUM_GSO
+	}
+	if d.fetchBooleanAnnotation("AfPacketQdiscBypass", false /* default */, uplinkSpec) {
+		intf.Flags |= af_packet.AF_PACKET_API_FLAG_QDISC_BYPASS
+	}
+	if !d.fetchBooleanAnnotation("AfPacketUseV3", false /* default */, uplinkSpec) {
+		intf.Flags |= af_packet.AF_PACKET_API_FLAG_VERSION_2
 	}
 	swIfIndex, err := vpp.CreateAfPacket(&intf)
 	if err != nil {

diff --git a/vpp-manager/uplink/af_xdp.go b/vpp-manager/uplink/af_xdp.go
@@ -129,7 +129,7 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid)
 	if err != nil {
 		return errors.Wrap(err, "Moving uplink in NS failed")

diff --git a/vpp-manager/uplink/avf.go b/vpp-manager/uplink/avf.go
@@ -136,7 +136,7 @@ func (d *AVFDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	if d.pfPCI != "" {
 		/* We were passed a PF, move it to vpp's NS so it doesn't
 		   conflict with vpptap0 */

diff --git a/vpp-manager/uplink/common.go b/vpp-manager/uplink/common.go
@@ -49,7 +49,7 @@ type UplinkDriverData struct {
 
 type UplinkDriver interface {
 	PreconfigureLinux() error
-	CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) error
+	CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) error
 	RestoreLinux(allInterfacesPhysical bool)
 	IsSupported(warn bool) bool
 	GetName() string

diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go
@@ -76,7 +76,7 @@ func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	// If interface is still in the host, move it to vpp netns to allow creation of the tap
 	err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid)
 	if err != nil {

diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go
@@ -155,7 +155,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	// Nothing to do VPP autocreates on startup
 	// refusing to run on secondary interfaces as we have no way to figure out the sw_if_index
 	if !d.spec.GetIsMain() {

diff --git a/vpp-manager/uplink/rdma.go b/vpp-manager/uplink/rdma.go
@@ -68,7 +68,7 @@ func (d *RDMADriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	intf := types.RDMAInterface{
 		GenericVppInterface: d.getGenericVppInterface(),
 	}

diff --git a/vpp-manager/uplink/virtio.go b/vpp-manager/uplink/virtio.go
@@ -111,7 +111,7 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	intf := types.VirtioInterface{
 		GenericVppInterface: d.getGenericVppInterface(),
 		PciId:               d.conf.PciId,

diff --git a/vpp-manager/uplink/vmxnet3.go b/vpp-manager/uplink/vmxnet3.go
@@ -85,7 +85,7 @@ func (d *Vmxnet3Driver) RestoreLinux(allInterfacesPhysical bool) {
 	d.restoreLinuxIfConf(link)
 }
 
-func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) {
+func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) {
 	intf := types.Vmxnet3Interface{
 		GenericVppInterface: d.getGenericVppInterface(),
 		EnableGso:           d.params.EnableGSO,

diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go
@@ -759,7 +759,7 @@ func (v *VppRunner) runVpp() (err error) {
 	}
 
 	for idx := 0; idx < len(v.params.UplinksSpecs); idx++ {
-		err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid)
+		err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid, &v.params.UplinksSpecs[idx])
 		if err != nil {
 			terminateVpp("Error creating main interface %s (SIGINT %d): %v", v.params.UplinksSpecs[idx].InterfaceName, vppProcess.Pid, err)
 			v.vpp.Close()

diff --git a/vpplink/af_packet.go b/vpplink/af_packet.go
@@ -37,7 +37,7 @@ func (v *VppLink) CreateAfPacket(intf *types.AfPacketInterface) (swIfIndex uint3
 		TxFrameSize:      uint32(1024 * 8 * 8),
 		NumRxQueues:      uint16(intf.NumRxQueues),
 		NumTxQueues:      uint16(intf.NumTxQueues),
-		Flags:            af_packet.AF_PACKET_API_FLAG_VERSION_2 | af_packet.AF_PACKET_API_FLAG_CKSUM_GSO | af_packet.AF_PACKET_API_FLAG_QDISC_BYPASS,
+		Flags:            intf.Flags,
 	}
 	if intf.HardwareAddr != nil {
 		request.UseRandomHwAddr = false

diff --git a/vpplink/types/vpp_interface.go b/vpplink/types/vpp_interface.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/af_packet"
 	interfaces "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface"
 	"github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types"
 )
@@ -92,6 +93,7 @@ type VppXDPInterface struct {
 
 type AfPacketInterface struct {
 	GenericVppInterface
+	Flags af_packet.AfPacketFlags
 }
 
 type VirtioInterface struct {