From 5e69e3cc5ae6801b661891fadb8058067c618fd7 Mon Sep 17 00:00:00 2001 From: wenyingd Date: Mon, 15 Nov 2021 10:31:39 +0800 Subject: [PATCH] Use OVS Interface external_ids to save Antrea Interface type Introduce Antrea Interface Type to map the OVS port to Antrea created interfaces, and use OVS Interface external_ids field to save the configuration. This type is helpful for Antrea to define more inteface types. Signed-off-by: wenyingd --- pkg/agent/agent.go | 20 +++++------ pkg/agent/agent_linux.go | 2 +- pkg/agent/agent_windows.go | 3 +- pkg/agent/cniserver/pod_configuration.go | 4 +-- pkg/ovs/ovsconfig/interfaces.go | 4 +-- pkg/ovs/ovsconfig/ovs_client.go | 39 +++++++++++++-------- pkg/ovs/ovsconfig/ovs_schema.go | 1 + pkg/ovs/ovsconfig/testing/mock_ovsconfig.go | 16 ++++----- test/integration/agent/cniserver_test.go | 4 +-- test/integration/ovs/ovs_client_test.go | 28 ++++++++------- 10 files changed, 67 insertions(+), 54 deletions(-) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index f2455027e10..245e66bdc66 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -227,15 +227,14 @@ func (i *Initializer) initInterfaceStore() error { } ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts)) - uplinkIfName := i.nodeConfig.UplinkNetConfig.Name for index := range ovsPorts { port := &ovsPorts[index] ovsPort := &interfacestore.OVSPortConfig{ PortUUID: port.UUID, OFPort: port.OFPort} var intf *interfacestore.InterfaceConfig - switch { - case port.OFPort == config.HostGatewayOFPort: + switch port.AntreaIFType { + case ovsconfig.AntreaGateway: intf = &interfacestore.InterfaceConfig{ Type: interfacestore.GatewayInterface, InterfaceName: port.Name, @@ -246,19 +245,13 @@ func (i *Initializer) initInterfaceStore() error { // Set the gateway interface name to the discovered name. i.hostGateway = intf.InterfaceName } - case port.Name == uplinkIfName: + case ovsconfig.AntreaUplink: intf = &interfacestore.InterfaceConfig{ Type: interfacestore.UplinkInterface, InterfaceName: port.Name, OVSPortConfig: ovsPort, } - case port.IFType == ovsconfig.GeneveTunnel: - fallthrough - case port.IFType == ovsconfig.VXLANTunnel: - fallthrough - case port.IFType == ovsconfig.GRETunnel: - fallthrough - case port.IFType == ovsconfig.STTTunnel: + case ovsconfig.AntreaTunnel: intf = noderoute.ParseTunnelInterfaceConfig(port, ovsPort) if intf != nil && port.OFPort == config.DefaultTunOFPort && intf.InterfaceName != i.nodeConfig.DefaultTunName { @@ -267,6 +260,9 @@ func (i *Initializer) initInterfaceStore() error { // Set the default tunnel interface name to the discovered name. i.nodeConfig.DefaultTunName = intf.InterfaceName } + case ovsconfig.AntreaHostIF: + // Not load the host interface. + continue default: // The port should be for a container interface. intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort, true) @@ -556,7 +552,7 @@ func (i *Initializer) setupGatewayInterface() error { gatewayIface, portExists := i.ifaceStore.GetInterface(i.hostGateway) if !portExists { klog.V(2).Infof("Creating gateway port %s on OVS bridge", i.hostGateway) - gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, nil) + gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, ovsconfig.AntreaGateway, nil) if err != nil { klog.Errorf("Failed to create gateway port %s on OVS bridge: %v", i.hostGateway, err) return err diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 3797249e3ff..c43825e68ea 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -114,7 +114,7 @@ func (i *Initializer) prepareOVSBridge() error { } else { // OVS does not receive "ofport_request" param when creating local port, so here use config.AutoAssignedOFPort=0 // to ignore this param. - if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, nil); err != nil { + if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, ovsconfig.AntreaHostIF, nil); err != nil { return err } } diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 3a6ed8f2342..7b75a716757 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -28,6 +28,7 @@ import ( "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/ovs/ovsctl" "antrea.io/antrea/pkg/util/ip" ) @@ -144,7 +145,7 @@ func (i *Initializer) prepareOVSBridge() error { } else { // OVS does not receive "ofport_request" param when creating local port, so here use config.AutoAssignedOFPort=0 // to ignore this param. - if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, nil); err != nil { + if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, ovsconfig.AntreaHostIF, nil); err != nil { return err } } diff --git a/pkg/agent/cniserver/pod_configuration.go b/pkg/agent/cniserver/pod_configuration.go index bf7a6e4e34e..0c9152cde71 100644 --- a/pkg/agent/cniserver/pod_configuration.go +++ b/pkg/agent/cniserver/pod_configuration.go @@ -259,9 +259,9 @@ func (pc *podConfigurator) createOVSPort(ovsPortName string, ovsAttachInfo map[s var err error switch pc.ifConfigurator.getOVSInterfaceType() { case internalOVSInterfaceType: - portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, ovsAttachInfo) + portUUID, err = pc.ovsBridgeClient.CreateInternalPort(ovsPortName, 0, ovsconfig.AntreaContainer, ovsAttachInfo) default: - portUUID, err = pc.ovsBridgeClient.CreatePort(ovsPortName, ovsPortName, ovsAttachInfo) + portUUID, err = pc.ovsBridgeClient.CreatePort(ovsPortName, ovsPortName, ovsconfig.AntreaContainer, ovsAttachInfo) } if err != nil { klog.Errorf("Failed to add OVS port %s, remove from local cache: %v", ovsPortName, err) diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index 941f6fa8e63..c3489fcf2d6 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -36,8 +36,8 @@ type OVSBridgeClient interface { SetDatapathID(datapathID string) Error GetInterfaceOptions(name string) (map[string]string, Error) SetInterfaceOptions(name string, options map[string]interface{}) Error - CreatePort(name, ifDev string, externalIDs map[string]interface{}) (string, Error) - CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) + CreatePort(name, ifDev string, antreaType string, externalIDs map[string]interface{}) (string, Error) + CreateInternalPort(name string, ofPortRequest int32, antreaType string, externalIDs map[string]interface{}) (string, Error) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, psk string, externalIDs map[string]interface{}) (string, Error) CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index 8659cd807aa..12a9c79f377 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -40,11 +40,13 @@ type OVSPortData struct { UUID string Name string // Interface type. - IFType string - IFName string - OFPort int32 - ExternalIDs map[string]string - Options map[string]string + IFType string + // The interface type defined by Antrea, which is saved in Interface external_ids. + AntreaIFType string + IFName string + OFPort int32 + ExternalIDs map[string]string + Options map[string]string } const ( @@ -56,6 +58,13 @@ const ( // Maximum allowed value of ofPortRequest. ofPortRequestMax = 65279 hardwareOffload = "hw-offload" + + AntreaIFTypeKey = "antrea-type" + AntreaGateway = "gateway" + AntreaContainer = "container" + AntreaTunnel = "tunnel" + AntreaUplink = "uplink" + AntreaHostIF = "hostLink" ) // NewOVSDBConnectionUDS connects to the OVSDB server on the UNIX domain socket @@ -361,11 +370,11 @@ func (br *OVSBridge) DeletePort(portUUID string) Error { // If externalIDs is not empty, the map key/value pairs will be set to the // port's external_ids. // If ofPortRequest is not zero, it will be passed to the OVS port creation. -func (br *OVSBridge) CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) { +func (br *OVSBridge) CreateInternalPort(name string, ofPortRequest int32, antreaType string, externalIDs map[string]interface{}) (string, Error) { if ofPortRequest < 0 || ofPortRequest > ofPortRequestMax { return "", newInvalidArgumentsError(fmt.Sprint("invalid ofPortRequest value: ", ofPortRequest)) } - return br.createPort(name, name, "internal", ofPortRequest, externalIDs, nil) + return br.createPort(name, name, "internal", antreaType, ofPortRequest, externalIDs, nil) } // CreateTunnelPort creates a tunnel port with the specified name and type on @@ -436,7 +445,7 @@ func (br *OVSBridge) createTunnelPort( options["csum"] = "true" } - return br.createPort(name, name, string(tunnelType), ofPortRequest, externalIDs, options) + return br.createPort(name, name, string(tunnelType), AntreaTunnel, ofPortRequest, externalIDs, options) } // GetInterfaceOptions returns the options of the provided interface. @@ -508,18 +517,18 @@ func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, // CreateUplinkPort creates uplink port. func (br *OVSBridge) CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) { - return br.createPort(name, name, "", ofPortRequest, externalIDs, nil) + return br.createPort(name, name, "", AntreaUplink, ofPortRequest, externalIDs, nil) } // CreatePort creates a port with the specified name on the bridge, and connects // the interface specified by ifDev to the port. // If externalIDs is not empty, the map key/value pairs will be set to the // port's external_ids. -func (br *OVSBridge) CreatePort(name, ifDev string, externalIDs map[string]interface{}) (string, Error) { - return br.createPort(name, ifDev, "", 0, externalIDs, nil) +func (br *OVSBridge) CreatePort(name, ifDev string, antreaType string, externalIDs map[string]interface{}) (string, Error) { + return br.createPort(name, ifDev, "", antreaType, 0, externalIDs, nil) } -func (br *OVSBridge) createPort(name, ifName, ifType string, ofPortRequest int32, externalIDs, options map[string]interface{}) (string, Error) { +func (br *OVSBridge) createPort(name, ifName, ifType string, antreaType string, ofPortRequest int32, externalIDs, options map[string]interface{}) (string, Error) { var externalIDMap []interface{} var optionMap []interface{} @@ -537,6 +546,7 @@ func (br *OVSBridge) createPort(name, ifName, ifType string, ofPortRequest int32 Type: ifType, OFPortRequest: ofPortRequest, Options: optionMap, + ExternalIDs: helpers.MakeOVSDBMap(map[string]interface{}{AntreaIFTypeKey: antreaType}), } ifNamedUUID := tx.Insert(dbtransaction.Insert{ Table: "Interface", @@ -640,6 +650,7 @@ func buildPortDataCommon(port, intf map[string]interface{}, portData *OVSPortDat } else { // ofport not assigned by OVS yet portData.OFPort = 0 } + portData.AntreaIFType = buildMapFromOVSDBMap(intf["external_ids"].([]interface{}))[AntreaIFTypeKey] } // GetPortData retrieves port data given the OVS port UUID and interface name. @@ -655,7 +666,7 @@ func (br *OVSBridge) GetPortData(portUUID, ifName string) (*OVSPortData, Error) }) tx.Select(dbtransaction.Select{ Table: "Interface", - Columns: []string{"_uuid", "type", "ofport", "options"}, + Columns: []string{"_uuid", "type", "ofport", "options", "external_ids"}, Where: [][]interface{}{{"name", "==", ifName}}, }) @@ -708,7 +719,7 @@ func (br *OVSBridge) GetPortList() ([]OVSPortData, Error) { }) tx.Select(dbtransaction.Select{ Table: "Interface", - Columns: []string{"_uuid", "type", "name", "ofport", "options"}, + Columns: []string{"_uuid", "type", "name", "ofport", "options", "external_ids"}, }) res, err, temporary := tx.Commit() diff --git a/pkg/ovs/ovsconfig/ovs_schema.go b/pkg/ovs/ovsconfig/ovs_schema.go index 689d097c11a..084a4633778 100644 --- a/pkg/ovs/ovsconfig/ovs_schema.go +++ b/pkg/ovs/ovsconfig/ovs_schema.go @@ -31,4 +31,5 @@ type Interface struct { Type string `json:"type,omitempty"` OFPortRequest int32 `json:"ofport_request,omitempty"` Options []interface{} `json:"options,omitempty"` + ExternalIDs []interface{} `json:"external_ids,omitempty"` } diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index a519d0b6492..536652afbcf 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -77,33 +77,33 @@ func (mr *MockOVSBridgeClientMockRecorder) Create() *gomock.Call { } // CreateInternalPort mocks base method -func (m *MockOVSBridgeClient) CreateInternalPort(arg0 string, arg1 int32, arg2 map[string]interface{}) (string, ovsconfig.Error) { +func (m *MockOVSBridgeClient) CreateInternalPort(arg0 string, arg1 int32, arg2 string, arg3 map[string]interface{}) (string, ovsconfig.Error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateInternalPort", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreateInternalPort", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(string) ret1, _ := ret[1].(ovsconfig.Error) return ret0, ret1 } // CreateInternalPort indicates an expected call of CreateInternalPort -func (mr *MockOVSBridgeClientMockRecorder) CreateInternalPort(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockOVSBridgeClientMockRecorder) CreateInternalPort(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInternalPort", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateInternalPort), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateInternalPort", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateInternalPort), arg0, arg1, arg2, arg3) } // CreatePort mocks base method -func (m *MockOVSBridgeClient) CreatePort(arg0, arg1 string, arg2 map[string]interface{}) (string, ovsconfig.Error) { +func (m *MockOVSBridgeClient) CreatePort(arg0, arg1, arg2 string, arg3 map[string]interface{}) (string, ovsconfig.Error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreatePort", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "CreatePort", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(string) ret1, _ := ret[1].(ovsconfig.Error) return ret0, ret1 } // CreatePort indicates an expected call of CreatePort -func (mr *MockOVSBridgeClientMockRecorder) CreatePort(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockOVSBridgeClientMockRecorder) CreatePort(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePort", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreatePort), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreatePort", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreatePort), arg0, arg1, arg2, arg3) } // CreateTunnelPort mocks base method diff --git a/test/integration/agent/cniserver_test.go b/test/integration/agent/cniserver_test.go index 0b7dabeffb3..88e6e06ab8d 100644 --- a/test/integration/agent/cniserver_test.go +++ b/test/integration/agent/cniserver_test.go @@ -606,7 +606,7 @@ func cmdAddDelCheckTest(testNS ns.NetNS, tc testCase, dataDir string) { // Mock ovs output while get ovs port external configuration ovsPortname := util.GenerateContainerInterfaceName(testPod, testPodNamespace, ContainerID) ovsPortUUID := uuid.New().String() - ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any()).Return(ovsPortUUID, nil).AnyTimes() + ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any(), mock.Any()).Return(ovsPortUUID, nil).AnyTimes() ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(int32(10), nil).AnyTimes() ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, mock.Any(), mock.Any(), mock.Any()).Return(nil) @@ -826,7 +826,7 @@ func TestCNIServerChaining(t *testing.T) { // Pod port expectations orderedCalls = append(orderedCalls, routeMock.EXPECT().MigrateRoutesToGw(hostVeth.Name), - ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any()).Return(ovsPortUUID, nil), + ovsServiceMock.EXPECT().CreatePort(ovsPortname, ovsPortname, mock.Any(), mock.Any()).Return(ovsPortUUID, nil), ovsServiceMock.EXPECT().GetOFPort(ovsPortname).Return(testContainerOFPort, nil), ofServiceMock.EXPECT().InstallPodFlows(ovsPortname, []net.IP{podIP}, containerIntf.HardwareAddr, mock.Any()), ) diff --git a/test/integration/ovs/ovs_client_test.go b/test/integration/ovs/ovs_client_test.go index 141195a26ba..072c5a347fe 100644 --- a/test/integration/ovs/ovs_client_test.go +++ b/test/integration/ovs/ovs_client_test.go @@ -113,10 +113,10 @@ func TestOVSBridge(t *testing.T) { deleteAllPorts(t, data.br) checkPorts(0) - uuid1 := testCreatePort(t, data.br, "p1", "internal") - uuid2 := testCreatePort(t, data.br, "p2", "") - uuid3 := testCreatePort(t, data.br, "p3", "vxlan") - uuid4 := testCreatePort(t, data.br, "p4", "geneve") + uuid1 := testCreatePort(t, data.br, "p1", "internal", ovsconfig.AntreaContainer) + uuid2 := testCreatePort(t, data.br, "p2", "", ovsconfig.AntreaGateway) + uuid3 := testCreatePort(t, data.br, "p3", "vxlan", "") + uuid4 := testCreatePort(t, data.br, "p4", "geneve", "") checkPorts(4) @@ -127,10 +127,10 @@ func TestOVSBridge(t *testing.T) { checkPorts(0) - testCreatePort(t, data.br, "p1", "internal") - testCreatePort(t, data.br, "p2", "") - testCreatePort(t, data.br, "p3", "vxlan") - testCreatePort(t, data.br, "p4", "geneve") + testCreatePort(t, data.br, "p1", "internal", ovsconfig.AntreaContainer) + testCreatePort(t, data.br, "p2", "", ovsconfig.AntreaGateway) + testCreatePort(t, data.br, "p3", "vxlan", "") + testCreatePort(t, data.br, "p4", "geneve", "") checkPorts(4) @@ -148,7 +148,7 @@ func TestOVSDeletePortIdempotent(t *testing.T) { deleteAllPorts(t, data.br) - uuid := testCreatePort(t, data.br, "p1", "internal") + uuid := testCreatePort(t, data.br, "p1", "internal", ovsconfig.AntreaContainer) testDeletePort(t, data.br, uuid) testDeletePort(t, data.br, uuid) } @@ -269,7 +269,7 @@ func deleteAllPorts(t *testing.T, br *ovsconfig.OVSBridge) { var ofPortRequest int32 = 1 -func testCreatePort(t *testing.T, br *ovsconfig.OVSBridge, name string, ifType string) string { +func testCreatePort(t *testing.T, br *ovsconfig.OVSBridge, name string, ifType string, antreaIFType string) string { var err error var uuid string var externalIDs map[string]interface{} @@ -278,10 +278,10 @@ func testCreatePort(t *testing.T, br *ovsconfig.OVSBridge, name string, ifType s switch ifType { case "": externalIDs = map[string]interface{}{"k1": "v1", "k2": "v2"} - uuid, err = br.CreatePort(name, name, externalIDs) + uuid, err = br.CreatePort(name, name, antreaIFType, externalIDs) case "internal": externalIDs = map[string]interface{}{"k1": "v1", "k2": "v2"} - uuid, err = br.CreateInternalPort(name, ofPortRequest, externalIDs) + uuid, err = br.CreateInternalPort(name, ofPortRequest, antreaIFType, externalIDs) case "vxlan": externalIDs = map[string]interface{}{} uuid, err = br.CreateTunnelPort(name, ovsconfig.VXLANTunnel, ofPortRequest) @@ -310,6 +310,10 @@ func testCreatePort(t *testing.T, br *ovsconfig.OVSBridge, name string, ifType s if ifType != "" { assert.Equal(t, ofPort, port.OFPort) } + if antreaIFType == "" { + antreaIFType = ovsconfig.AntreaTunnel + } + assert.Equal(t, antreaIFType, port.AntreaIFType) for k, v := range externalIDs { rv, ok := port.ExternalIDs[k]