From 82c88a14e5e0e3cc5d7f70c52cdbc01c999d3a42 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 20 Dec 2022 15:16:06 +0100 Subject: [PATCH] vpp-manager: Add uplink annotations, default disable qdisc bypass This adds an annotation map on the uplinkConfiguration object in the json definition allowing to change v2/v3 version for af_packet and the use of qdisc bypass or not. We now default to qdisc bypass disables as this seems to be related to issues on EC2. But this is now exposed in the configuration. Signed-off-by: Nathan Skrzypczak --- config/config/config.go | 3 ++- vpp-manager/uplink/af_packet.go | 34 ++++++++++++++++++++++++++++----- vpp-manager/uplink/af_xdp.go | 2 +- vpp-manager/uplink/avf.go | 2 +- vpp-manager/uplink/common.go | 2 +- vpp-manager/uplink/default.go | 2 +- vpp-manager/uplink/dpdk.go | 2 +- vpp-manager/uplink/rdma.go | 2 +- vpp-manager/uplink/virtio.go | 2 +- vpp-manager/uplink/vmxnet3.go | 2 +- vpp-manager/vpp_runner.go | 2 +- vpplink/af_packet.go | 2 +- vpplink/types/vpp_interface.go | 2 ++ 13 files changed, 43 insertions(+), 16 deletions(-) diff --git a/config/config/config.go b/config/config/config.go index 69c34a21..c08decd3 100644 --- a/config/config/config.go +++ b/config/config/config.go @@ -43,7 +43,8 @@ type InterfaceSpec struct { TxQueueSize int `json:"txqsz"` IsL3 *bool `json:"isl3"` /* "interrupt" "adaptive" or "polling" mode */ - RxMode types.RxMode `json:"rxMode"` + RxMode types.RxMode `json:"rxMode"` + Annotations map[string]string `json:"annotations"` } func (i *InterfaceSpec) GetIsL3(isMemif bool) bool { diff --git a/vpp-manager/uplink/af_packet.go b/vpp-manager/uplink/af_packet.go index 1c37fe5e..0ee01785 100644 --- a/vpp-manager/uplink/af_packet.go +++ b/vpp-manager/uplink/af_packet.go @@ -16,13 +16,17 @@ package uplink import ( + "strconv" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "github.com/projectcalico/vpp-dataplane/config/config" "github.com/projectcalico/vpp-dataplane/vpp-manager/utils" "github.com/projectcalico/vpp-dataplane/vpplink" + "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/af_packet" "github.com/projectcalico/vpp-dataplane/vpplink/types" - log "github.com/sirupsen/logrus" - "github.com/vishvananda/netlink" ) type AFPacketDriver struct { @@ -75,14 +79,34 @@ func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *AFPacketDriver) fetchBooleanAnnotation(annotation string, defaultValue bool, uplinkSpec *config.UplinkInterfaceSpec) bool { + spec, found := uplinkSpec.Annotations[annotation] + if !found { + return defaultValue + } + b, err := strconv.ParseBool(spec) + if err != nil { + log.WithError(err).Errorf("Error parsing annotation %s '%s'", annotation, spec) + return defaultValue + } + return b +} + +func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } - intf := types.AfPacketInterface{ - GenericVppInterface: d.getGenericVppInterface(), + intf := types.AfPacketInterface{GenericVppInterface: d.getGenericVppInterface()} + if d.params.EnableGSO { + intf.Flags |= af_packet.AF_PACKET_API_FLAG_CKSUM_GSO + } + if d.fetchBooleanAnnotation("AfPacketQdiscBypass", false /* default */, uplinkSpec) { + intf.Flags |= af_packet.AF_PACKET_API_FLAG_QDISC_BYPASS + } + if !d.fetchBooleanAnnotation("AfPacketUseV3", false /* default */, uplinkSpec) { + intf.Flags |= af_packet.AF_PACKET_API_FLAG_VERSION_2 } swIfIndex, err := vpp.CreateAfPacket(&intf) if err != nil { diff --git a/vpp-manager/uplink/af_xdp.go b/vpp-manager/uplink/af_xdp.go index 9d4daeea..259a9009 100644 --- a/vpp-manager/uplink/af_xdp.go +++ b/vpp-manager/uplink/af_xdp.go @@ -129,7 +129,7 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") diff --git a/vpp-manager/uplink/avf.go b/vpp-manager/uplink/avf.go index 0196adb6..ef6aac1a 100644 --- a/vpp-manager/uplink/avf.go +++ b/vpp-manager/uplink/avf.go @@ -136,7 +136,7 @@ func (d *AVFDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { if d.pfPCI != "" { /* We were passed a PF, move it to vpp's NS so it doesn't conflict with vpptap0 */ diff --git a/vpp-manager/uplink/common.go b/vpp-manager/uplink/common.go index f340bf1e..e5001e0c 100644 --- a/vpp-manager/uplink/common.go +++ b/vpp-manager/uplink/common.go @@ -49,7 +49,7 @@ type UplinkDriverData struct { type UplinkDriver interface { PreconfigureLinux() error - CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) error + CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) error RestoreLinux(allInterfacesPhysical bool) IsSupported(warn bool) bool GetName() string diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go index efdbc2f8..d3ab58d8 100644 --- a/vpp-manager/uplink/default.go +++ b/vpp-manager/uplink/default.go @@ -76,7 +76,7 @@ func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // If interface is still in the host, move it to vpp netns to allow creation of the tap err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) if err != nil { diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go index 4d3ebdb7..c65a832c 100644 --- a/vpp-manager/uplink/dpdk.go +++ b/vpp-manager/uplink/dpdk.go @@ -155,7 +155,7 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // Nothing to do VPP autocreates on startup // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index if !d.spec.GetIsMain() { diff --git a/vpp-manager/uplink/rdma.go b/vpp-manager/uplink/rdma.go index bfd85766..af9954c5 100644 --- a/vpp-manager/uplink/rdma.go +++ b/vpp-manager/uplink/rdma.go @@ -68,7 +68,7 @@ func (d *RDMADriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { intf := types.RDMAInterface{ GenericVppInterface: d.getGenericVppInterface(), } diff --git a/vpp-manager/uplink/virtio.go b/vpp-manager/uplink/virtio.go index 9c66f00c..e3883203 100644 --- a/vpp-manager/uplink/virtio.go +++ b/vpp-manager/uplink/virtio.go @@ -111,7 +111,7 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { intf := types.VirtioInterface{ GenericVppInterface: d.getGenericVppInterface(), PciId: d.conf.PciId, diff --git a/vpp-manager/uplink/vmxnet3.go b/vpp-manager/uplink/vmxnet3.go index cb3d0c17..fb9b4cdb 100644 --- a/vpp-manager/uplink/vmxnet3.go +++ b/vpp-manager/uplink/vmxnet3.go @@ -85,7 +85,7 @@ func (d *Vmxnet3Driver) RestoreLinux(allInterfacesPhysical bool) { d.restoreLinuxIfConf(link) } -func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int) (err error) { +func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { intf := types.Vmxnet3Interface{ GenericVppInterface: d.getGenericVppInterface(), EnableGso: d.params.EnableGSO, diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index 32478d6d..f87c7f36 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -759,7 +759,7 @@ func (v *VppRunner) runVpp() (err error) { } for idx := 0; idx < len(v.params.UplinksSpecs); idx++ { - err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid) + err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid, &v.params.UplinksSpecs[idx]) if err != nil { terminateVpp("Error creating main interface %s (SIGINT %d): %v", v.params.UplinksSpecs[idx].InterfaceName, vppProcess.Pid, err) v.vpp.Close() diff --git a/vpplink/af_packet.go b/vpplink/af_packet.go index 02eef0f6..dae7763e 100644 --- a/vpplink/af_packet.go +++ b/vpplink/af_packet.go @@ -37,7 +37,7 @@ func (v *VppLink) CreateAfPacket(intf *types.AfPacketInterface) (swIfIndex uint3 TxFrameSize: uint32(1024 * 8 * 8), NumRxQueues: uint16(intf.NumRxQueues), NumTxQueues: uint16(intf.NumTxQueues), - Flags: af_packet.AF_PACKET_API_FLAG_VERSION_2 | af_packet.AF_PACKET_API_FLAG_CKSUM_GSO | af_packet.AF_PACKET_API_FLAG_QDISC_BYPASS, + Flags: intf.Flags, } if intf.HardwareAddr != nil { request.UseRandomHwAddr = false diff --git a/vpplink/types/vpp_interface.go b/vpplink/types/vpp_interface.go index 9df75e8f..715bf438 100644 --- a/vpplink/types/vpp_interface.go +++ b/vpplink/types/vpp_interface.go @@ -23,6 +23,7 @@ import ( "github.com/pkg/errors" + "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/af_packet" interfaces "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface" "github.com/projectcalico/vpp-dataplane/vpplink/binapi/vppapi/interface_types" ) @@ -92,6 +93,7 @@ type VppXDPInterface struct { type AfPacketInterface struct { GenericVppInterface + Flags af_packet.AfPacketFlags } type VirtioInterface struct {