From 35583b485bd98336bb7ff6e76ea1cb2c072926a2 Mon Sep 17 00:00:00 2001 From: Wenqi Qiu Date: Wed, 20 Apr 2022 14:01:59 +0800 Subject: [PATCH] Translate the target device to the OVS port(Not test yet) Signed-off-by: Wenqi Qiu --- cmd/antrea-agent/agent.go | 2 +- .../controller/trafficcontrol/controller.go | 129 ++++++++++++++---- .../trafficcontrol/controller_test.go | 88 ++++++------ pkg/agent/openflow/client.go | 4 +- pkg/agent/openflow/testing/mock_openflow.go | 2 +- 5 files changed, 158 insertions(+), 67 deletions(-) diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index c3d32187fc1..9747aeff85a 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -528,7 +528,7 @@ func run(o *Options) error { } if features.DefaultFeatureGate.Enabled(features.TrafficControl) { - tcController := trafficcontrol.NewTrafficControlController(nodeConfig.Name, ofClient, ifaceStore, trafficControlInformer, localPodInformer, namespaceInformer) + tcController := trafficcontrol.NewTrafficControlController(nodeConfig.Name, ofClient, ifaceStore, ovsBridgeClient, trafficControlInformer, localPodInformer, namespaceInformer) go tcController.Run(stopCh) } diff --git a/pkg/agent/controller/trafficcontrol/controller.go b/pkg/agent/controller/trafficcontrol/controller.go index 41e3a1b5172..6ad5ac4dc80 100644 --- a/pkg/agent/controller/trafficcontrol/controller.go +++ b/pkg/agent/controller/trafficcontrol/controller.go @@ -37,6 +37,7 @@ import ( "antrea.io/antrea/pkg/apis/crd/v1alpha2" trafficControlinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" trafficControllisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/ovs/ovsconfig" ) const ( @@ -58,7 +59,8 @@ type tcState struct { type Controller struct { nodeName string - ofClient openflow.Client + ofClient openflow.Client + ovsBridgeClient ovsconfig.OVSBridgeClient interfaceStore interfacestore.InterfaceStore @@ -72,6 +74,8 @@ type Controller struct { tcStates map[string]*tcState tcStatesMutex sync.RWMutex + ovsPorts map[int32]ovsconfig.OVSPortData + trafficControlInformer cache.SharedIndexInformer // trafficControlLister trafficControlLister trafficControllisters.TrafficControlLister @@ -83,11 +87,12 @@ type Controller struct { } func NewTrafficControlController(nodeName string, ofClient openflow.Client, - interfaceStore interfacestore.InterfaceStore, tcInformer trafficControlinformers.TrafficControlInformer, - podInformer cache.SharedIndexInformer, namespaceInformer v12.NamespaceInformer) *Controller { + interfaceStore interfacestore.InterfaceStore, ovsBridgeClient ovsconfig.OVSBridgeClient, + tcInformer trafficControlinformers.TrafficControlInformer, podInformer cache.SharedIndexInformer, namespaceInformer v12.NamespaceInformer) *Controller { c := &Controller{ nodeName: nodeName, ofClient: ofClient, + ovsBridgeClient: ovsBridgeClient, interfaceStore: interfaceStore, trafficControlInformer: tcInformer.Informer(), trafficControlLister: tcInformer.Lister(), @@ -245,28 +250,103 @@ func (c *Controller) filterPods(appliedTo *v1alpha2.AppliedTo) ([]*v1.Pod, error return c.podLister.List(podSelector) } -func (c *Controller) getDevicePort(device *v1alpha2.Device) (uint32, error) { +func (c *Controller) getDevicePort(device *v1alpha2.OVSPort) (uint32, error) { + // TODOļ¼šNot test yet if device == nil || device.Name == "" { - return 0, fmt.Errorf("device is nil") + return 0, fmt.Errorf("device is invalid") } itf, ok := c.interfaceStore.GetInterfaceByName(device.Name) - if !ok { - return 0, fmt.Errorf("device interface not found %s", device.Name) - } - // TODO - switch { - case device.ERSPAN != nil: - return uint32(itf.OFPort), nil - case device.Port != nil: - return uint32(itf.OFPort), nil - case device.GRE != nil: - return uint32(itf.OFPort), nil - case device.GENEVE != nil: - return uint32(itf.OFPort), nil - case device.VXLAN != nil: + if ok { return uint32(itf.OFPort), nil } - return uint32(itf.OFPort), nil + switch device.Type { + case v1alpha2.OVSPortTypeInternal: + tunnelConfig := device.TunnelConfig + tunnelID := tunnelConfig.TunnelID + if port, ok := c.ovsPorts[int32(tunnelID)]; ok { + return uint32(port.OFPort), nil + } + portUUID, err := c.ovsBridgeClient.CreateTunnelPort(device.Name, ovsconfig.GeneveTunnel, int32(tunnelID)) + if err != nil { + return 0, err + } + klog.InfoS("Create internal port", "portUUID", portUUID) + case v1alpha2.OVSPortTypeDevice: + externalIDs := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, + } + portUUID, err := c.ovsBridgeClient.CreatePort(device.Name, device.Name, externalIDs) + if err != nil { + return 0, err + } + klog.InfoS("Create device", "portUUID", portUUID) + case v1alpha2.OVSPortTypeVXLAN: + tunnelConfig := device.TunnelConfig + tunnelID := tunnelConfig.TunnelID + remoteIP := tunnelConfig.RemoteIP + if port, ok := c.ovsPorts[int32(tunnelID)]; ok { + return uint32(port.OFPort), nil + } + portUUID, err := c.ovsBridgeClient.CreateTunnelPortExt( + device.Name, ovsconfig.VXLANTunnel, int32(tunnelID), false, "", remoteIP, "", nil) + if err != nil { + return 0, err + } + klog.InfoS("Create VXLANTunnel port", "portUUID", portUUID) + case v1alpha2.OVSPortTypeGENEVE: + tunnelConfig := device.TunnelConfig + tunnelID := tunnelConfig.TunnelID + remoteIP := tunnelConfig.RemoteIP + if port, ok := c.ovsPorts[int32(tunnelID)]; ok { + return uint32(port.OFPort), nil + } + portUUID, err := c.ovsBridgeClient.CreateTunnelPortExt( + device.Name, ovsconfig.GeneveTunnel, int32(tunnelID), false, "", remoteIP, "", nil) + if err != nil { + return 0, err + } + + klog.InfoS("Create GeneveTunnel port", "portUUID", portUUID) + case v1alpha2.OVSPortTypeGRE: + tunnelConfig := device.TunnelConfig + tunnelID := tunnelConfig.TunnelID + remoteIP := tunnelConfig.RemoteIP + if port, ok := c.ovsPorts[int32(tunnelID)]; ok { + return uint32(port.OFPort), nil + } + + portUUID, err := c.ovsBridgeClient.CreateTunnelPortExt( + device.Name, ovsconfig.GRETunnel, int32(tunnelID), false, "", remoteIP, "", nil) + if err != nil { + return 0, err + } + + klog.InfoS("Create GRETunnel port", "portUUID", portUUID) + case v1alpha2.OVSPortTypeERSPAN: + tunnelConfig := device.ERSPANConfig + tunnelID := tunnelConfig.TunnelID + remoteIP := tunnelConfig.RemoteIP + + klog.InfoS("", "tunnelID", tunnelID, "remoteIP", remoteIP) + } + ofPort, err := c.ovsBridgeClient.GetOFPort(device.Name, false) + if err != nil { + return 0, fmt.Errorf("failed to get of_port of device %s", device.Name) + } + return uint32(ofPort), nil +} + +func (c *Controller) listOVSPorts() error { + ovsPorts, err := c.ovsBridgeClient.GetPortList() + if err != nil { + return err + } + ports := make(map[int32]ovsconfig.OVSPortData, len(ovsPorts)) + for i := range ovsPorts { + ports[ovsPorts[i].OFPort] = ovsPorts[i] + } + c.ovsPorts = ports + return nil } func (c *Controller) syncTrafficControl(tcName string) (err error) { @@ -295,15 +375,18 @@ func (c *Controller) syncTrafficControl(tcName string) (err error) { if !exist { tcState = c.createTcState(tcName) var dstInPort, dstOutPort uint32 + if err := c.listOVSPorts(); err != nil { + return err + } - if dstInPort, err = c.getDevicePort(tc.Spec.ReturnDevice); err != nil && tc.Spec.Action != v1alpha2.ActionMirror { + if dstOutPort, err = c.getDevicePort(tc.Spec.ReturnPort); err != nil && tc.Spec.Action != v1alpha2.ActionMirror { return err } - if dstOutPort, err = c.getDevicePort(&tc.Spec.TargetDevice); err != nil { + if dstInPort, err = c.getDevicePort(&tc.Spec.TargetPort); err != nil { return err } if err := c.ofClient.InstallTrafficControlInitFlows(tcState.mark, tc.Spec.Action, dstInPort, dstOutPort); err != nil { - klog.ErrorS(err, "Installs the init flow failed", "Name", tcName) + klog.ErrorS(err, "Install the init flow failed", "Name", tcName) return err } } diff --git a/pkg/agent/controller/trafficcontrol/controller_test.go b/pkg/agent/controller/trafficcontrol/controller_test.go index 8f4da8923ed..105706a4647 100644 --- a/pkg/agent/controller/trafficcontrol/controller_test.go +++ b/pkg/agent/controller/trafficcontrol/controller_test.go @@ -34,18 +34,20 @@ import ( "antrea.io/antrea/pkg/apis/crd/v1alpha2" fakeversioned "antrea.io/antrea/pkg/client/clientset/versioned/fake" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" + ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" "antrea.io/antrea/pkg/util/k8s" ) type fakeController struct { *Controller - mockController *gomock.Controller - mockOFClient *openflowtest.MockClient - crdClient *fakeversioned.Clientset - crdInformerFactory crdinformers.SharedInformerFactory - client *fake.Clientset - informerFactory informers.SharedInformerFactory - localPodInformer cache.SharedIndexInformer + mockController *gomock.Controller + mockOFClient *openflowtest.MockClient + mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient + crdClient *fakeversioned.Clientset + crdInformerFactory crdinformers.SharedInformerFactory + client *fake.Clientset + informerFactory informers.SharedInformerFactory + localPodInformer cache.SharedIndexInformer } func newFakeController(t *testing.T, objects []runtime.Object, initObjects []runtime.Object) *fakeController { @@ -53,6 +55,8 @@ func newFakeController(t *testing.T, objects []runtime.Object, initObjects []run mockOFClient := openflowtest.NewMockClient(controller) + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(controller) + client := fake.NewSimpleClientset(objects...) crdClient := fakeversioned.NewSimpleClientset(initObjects...) crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, 0) @@ -73,7 +77,6 @@ func newFakeController(t *testing.T, objects []runtime.Object, initObjects []run addPodInterface(ifaceStore, "ns1", "pod1", 1) addPodInterface(ifaceStore, "ns2", "pod2", 2) - // TODO ifaceStore.AddInterface(&interfacestore.InterfaceConfig{ InterfaceName: "test-device", ContainerInterfaceConfig: &interfacestore.ContainerInterfaceConfig{PodName: "podName", PodNamespace: "podNamespace", ContainerID: "containerName"}, @@ -99,16 +102,17 @@ func newFakeController(t *testing.T, objects []runtime.Object, initObjects []run listOptions, ) - tcController := NewTrafficControlController("fakeNode", mockOFClient, ifaceStore, tcInformer, localPodInformer, nsInformer) + tcController := NewTrafficControlController("fakeNode", mockOFClient, ifaceStore, mockOVSBridgeClient, tcInformer, localPodInformer, nsInformer) return &fakeController{ - Controller: tcController, - mockController: controller, - mockOFClient: mockOFClient, - crdClient: crdClient, - crdInformerFactory: crdInformerFactory, - client: client, - informerFactory: informerFactory, - localPodInformer: localPodInformer, + Controller: tcController, + mockController: controller, + mockOFClient: mockOFClient, + mockOVSBridgeClient: mockOVSBridgeClient, + crdClient: crdClient, + crdInformerFactory: crdInformerFactory, + client: client, + informerFactory: informerFactory, + localPodInformer: localPodInformer, } } @@ -131,7 +135,7 @@ func TestSyncTrafficControl(t *testing.T) { name string tc *v1alpha2.TrafficControl localPods []*v1.Pod - expectedCalls func(mockOFClient *openflowtest.MockClient) + expectedCalls func(mockOFClient *openflowtest.MockClient, mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient) }{ { name: "create trafficControl", @@ -140,16 +144,17 @@ func TestSyncTrafficControl(t *testing.T) { Spec: v1alpha2.TrafficControlSpec{ AppliedTo: v1alpha2.AppliedTo{ PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}}, - Direction: v1alpha2.DirectionIngress, - Action: v1alpha2.ActionMirror, - TargetDevice: v1alpha2.Device{Name: "test-device"}, + Direction: v1alpha2.DirectionIngress, + Action: v1alpha2.ActionMirror, + TargetPort: v1alpha2.OVSPort{Name: "test-device"}, }}, localPods: []*v1.Pod{ newPod("ns1", "pod1", "fakeNode", map[string]string{"app": "foo"}), }, - expectedCalls: func(mockOFClient *openflowtest.MockClient) { + expectedCalls: func(mockOFClient *openflowtest.MockClient, mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient) { mockOFClient.EXPECT().InstallTrafficControlMarkFlows("trafficControl1", uint32(1), []uint32{1}, v1alpha2.DirectionIngress) - mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionMirror, uint32(0), uint32(3)) + mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionMirror, uint32(3), uint32(0)) + mockOVSBridgeClient.EXPECT().GetPortList() }, }, { @@ -159,17 +164,18 @@ func TestSyncTrafficControl(t *testing.T) { Spec: v1alpha2.TrafficControlSpec{ AppliedTo: v1alpha2.AppliedTo{ PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "foo"}}}, - Direction: v1alpha2.DirectionIngress, - Action: v1alpha2.ActionMirror, - TargetDevice: v1alpha2.Device{Name: "test-device"}, + Direction: v1alpha2.DirectionIngress, + Action: v1alpha2.ActionMirror, + TargetPort: v1alpha2.OVSPort{Name: "test-device"}, }}, localPods: []*v1.Pod{ newPod("ns1", "pod1", "fakeNode", map[string]string{"app": "foo"}), newPod("ns2", "pod2", "fakeNode", map[string]string{"app": "foo1"}), }, - expectedCalls: func(mockOFClient *openflowtest.MockClient) { + expectedCalls: func(mockOFClient *openflowtest.MockClient, mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient) { mockOFClient.EXPECT().InstallTrafficControlMarkFlows("trafficControl1", uint32(1), []uint32{1}, v1alpha2.DirectionIngress) - mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionMirror, uint32(0), uint32(3)) + mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionMirror, uint32(3), uint32(0)) + mockOVSBridgeClient.EXPECT().GetPortList() }, }, { @@ -177,17 +183,18 @@ func TestSyncTrafficControl(t *testing.T) { tc: &v1alpha2.TrafficControl{ ObjectMeta: metav1.ObjectMeta{Name: "trafficControl1", UID: "tc-uid"}, Spec: v1alpha2.TrafficControlSpec{ - Direction: v1alpha2.DirectionIngress, - Action: v1alpha2.ActionRedirect, - ReturnDevice: &v1alpha2.Device{Name: "test-device"}, - TargetDevice: v1alpha2.Device{Name: "test-device"}, + Direction: v1alpha2.DirectionIngress, + Action: v1alpha2.ActionRedirect, + ReturnPort: &v1alpha2.OVSPort{Name: "test-device"}, + TargetPort: v1alpha2.OVSPort{Name: "test-device"}, }}, localPods: []*v1.Pod{ newPod("ns1", "pod1", "fakeNode", map[string]string{"app": "foo"}), }, - expectedCalls: func(mockOFClient *openflowtest.MockClient) { + expectedCalls: func(mockOFClient *openflowtest.MockClient, mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient) { mockOFClient.EXPECT().InstallTrafficControlMarkFlows("trafficControl1", uint32(1), []uint32{1}, v1alpha2.DirectionIngress) mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionRedirect, uint32(3), uint32(3)) + mockOVSBridgeClient.EXPECT().GetPortList() }, }, { @@ -195,18 +202,19 @@ func TestSyncTrafficControl(t *testing.T) { tc: &v1alpha2.TrafficControl{ ObjectMeta: metav1.ObjectMeta{Name: "trafficControl1", UID: "tc-uid"}, Spec: v1alpha2.TrafficControlSpec{ - AppliedTo: v1alpha2.AppliedTo{}, - Direction: v1alpha2.DirectionIngress, - Action: v1alpha2.ActionRedirect, - ReturnDevice: &v1alpha2.Device{Name: "test-device"}, - TargetDevice: v1alpha2.Device{Name: "test-device"}, + AppliedTo: v1alpha2.AppliedTo{}, + Direction: v1alpha2.DirectionIngress, + Action: v1alpha2.ActionRedirect, + ReturnPort: &v1alpha2.OVSPort{Name: "test-device"}, + TargetPort: v1alpha2.OVSPort{Name: "test-device"}, }}, localPods: []*v1.Pod{ newPod("ns1", "pod1", "fakeNode", map[string]string{"app": "foo"}), }, - expectedCalls: func(mockOFClient *openflowtest.MockClient) { + expectedCalls: func(mockOFClient *openflowtest.MockClient, mockOVSBridgeClient *ovsconfigtest.MockOVSBridgeClient) { mockOFClient.EXPECT().InstallTrafficControlMarkFlows("trafficControl1", uint32(1), []uint32{1}, v1alpha2.DirectionIngress) mockOFClient.EXPECT().InstallTrafficControlInitFlows(uint32(1), v1alpha2.ActionRedirect, uint32(3), uint32(3)) + mockOVSBridgeClient.EXPECT().GetPortList() }, }, } @@ -230,7 +238,7 @@ func TestSyncTrafficControl(t *testing.T) { c.informerFactory.Start(stopCh) c.informerFactory.WaitForCacheSync(stopCh) - tt.expectedCalls(c.mockOFClient) + tt.expectedCalls(c.mockOFClient, c.mockOVSBridgeClient) assert.NoError(t, c.syncTrafficControl(tt.tc.Name)) }) diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index f046ac67926..a68209471eb 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -288,7 +288,7 @@ type Client interface { UninstallTrafficControlMarkFlows(name string) error // InstallTrafficControlInitFlows installs the init flows for the given mark value. - InstallTrafficControlInitFlows(mark uint32, action v1alpha2.Action, dstOfInPort, dstOfOutPort uint32) error + InstallTrafficControlInitFlows(mark uint32, action v1alpha2.TrafficControlAction, dstOfInPort, dstOfOutPort uint32) error // UninstallTrafficControlInitFlows removes the flows for the given mark value. UninstallTrafficControlInitFlows(mark uint32) error @@ -1154,7 +1154,7 @@ func (c *client) UninstallTrafficControlMarkFlows(name string) error { return nil } -func (c *client) InstallTrafficControlInitFlows(mark uint32, action v1alpha2.Action, dstOfInPort, dstOfOutPort uint32) error { +func (c *client) InstallTrafficControlInitFlows(mark uint32, action v1alpha2.TrafficControlAction, dstOfInPort, dstOfOutPort uint32) error { klog.InfoS("InstallTrafficControlInitFlows", "mark", mark, "action", action, "dstOfInPort", dstOfInPort, "dstOfOutPort", dstOfOutPort) return nil } diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 961d837385b..5e4597807b8 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -423,7 +423,7 @@ func (mr *MockClientMockRecorder) InstallTraceflowFlows(arg0, arg1, arg2, arg3, } // InstallTrafficControlInitFlows mocks base method -func (m *MockClient) InstallTrafficControlInitFlows(arg0 uint32, arg1 v1alpha2.Action, arg2, arg3 uint32) error { +func (m *MockClient) InstallTrafficControlInitFlows(arg0 uint32, arg1 v1alpha2.TrafficControlAction, arg2, arg3 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallTrafficControlInitFlows", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error)