From 445638684ebafd17c934fc99086c59e6d242fa2c Mon Sep 17 00:00:00 2001 From: wenyingd Date: Thu, 1 Dec 2022 11:06:35 +0800 Subject: [PATCH] Bugfix: Pod or gateway interface use a different MAC from the expectation This issue is found on Ubuntu 22.04: the network interface for antrea-gw0 is different from the one we used in OpenFlow rules. The reason is systemd-udev has modified the interface's MAC after it watches a new one is created. So if Antrea Agent reads interface's information before systemd-udev's modification, Antrea Agent would uses an incorrect value to install OpenFlow rules. To resolve the issue, 1. Agent generates a static MAC for antrea-gw0 or the interface used by Pod 2. Agent uses the generated MAC to create OVS internal port or veth pair To implement the logic, some code is copied from containernetworking/plugins/ip/link_linux latest versions to path thirdparty, this is to avoid unexpected issues introduced when bumping up the dependent libraries. Signed-off-by: wenyingd --- pkg/agent/agent.go | 24 +-- pkg/agent/agent_linux.go | 2 +- pkg/agent/agent_linux_test.go | 19 +++ pkg/agent/agent_test.go | 67 ++++++++ pkg/agent/agent_windows.go | 7 +- pkg/agent/agent_windows_test.go | 25 +++ .../interface_configuration_linux.go | 6 +- .../controller/trafficcontrol/controller.go | 2 +- .../interfacestore/interface_cache_test.go | 2 +- pkg/agent/interfacestore/types.go | 4 +- pkg/agent/util/net.go | 5 +- pkg/agent/util/net_linux.go | 11 +- pkg/agent/util/net_test.go | 12 ++ pkg/agent/util/net_windows.go | 13 +- pkg/ovs/ovsconfig/ovs_client.go | 18 ++- pkg/ovs/ovsconfig/ovs_client_test.go | 81 ++++++++++ test/integration/agent/net_linux_test.go | 2 +- test/integration/agent/net_windows_test.go | 2 +- .../containernetworking/ip/link_linux.go | 152 ++++++++++++++++++ 19 files changed, 418 insertions(+), 36 deletions(-) create mode 100644 pkg/agent/agent_linux_test.go create mode 100644 pkg/agent/agent_windows_test.go create mode 100644 third_party/containernetworking/ip/link_linux.go diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 3eca082724a..4bef3608f9e 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -77,6 +77,12 @@ var ( // getTransportIPNetDeviceByName is meant to be overridden for testing. getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName + + // setLinkUp is meant to be overridden for testing + setLinkUp = util.SetLinkUp + + // configureLinkAddresses is meant to be overridden for testing + configureLinkAddresses = util.ConfigureLinkAddresses ) // otherConfigKeysForIPsecCertificates are configurations added to OVS bridge when AuthenticationMode is "cert" and @@ -243,6 +249,7 @@ func (i *Initializer) initInterfaceStore() error { intf := &interfacestore.InterfaceConfig{ Type: interfacestore.GatewayInterface, InterfaceName: port.Name, + MAC: port.MAC, OVSPortConfig: ovsPort} if intf.InterfaceName != i.hostGateway { klog.Warningf("The discovered gateway interface name %s is different from the configured value: %s", @@ -639,7 +646,8 @@ func (i *Initializer) setupGatewayInterface() error { externalIDs := map[string]interface{}{ interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaGateway, } - gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, "", externalIDs) + mac := util.GenerateRandomMAC() + gwPortUUID, err := i.ovsBridgeClient.CreateInternalPort(i.hostGateway, config.HostGatewayOFPort, mac.String(), externalIDs) if err != nil { klog.ErrorS(err, "Failed to create gateway port on OVS bridge", "port", i.hostGateway) return err @@ -650,7 +658,7 @@ func (i *Initializer) setupGatewayInterface() error { return err } klog.InfoS("Allocated OpenFlow port for gateway interface", "port", i.hostGateway, "ofPort", gwPort) - gatewayIface = interfacestore.NewGatewayInterface(i.hostGateway) + gatewayIface = interfacestore.NewGatewayInterface(i.hostGateway, mac) gatewayIface.OVSPortConfig = &interfacestore.OVSPortConfig{PortUUID: gwPortUUID, OFPort: gwPort} i.ifaceStore.AddInterface(gatewayIface) } else { @@ -658,7 +666,7 @@ func (i *Initializer) setupGatewayInterface() error { } // Idempotent operation to set the gateway's MTU: we perform this operation regardless of - // whether or not the gateway interface already exists, as the desired MTU may change across + // whether the gateway interface already exists, as the desired MTU may change across // restarts. klog.V(4).Infof("Setting gateway interface %s MTU to %d", i.hostGateway, i.nodeConfig.NodeMTU) @@ -673,13 +681,12 @@ func (i *Initializer) setupGatewayInterface() error { } func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.InterfaceConfig) error { - var gwMAC net.HardwareAddr var gwLinkIdx int var err error // Host link might not be queried at once after creating OVS internal port; retry max 5 times with 1s // delay each time to ensure the link is ready. for retry := 0; retry < maxRetryForHostLink; retry++ { - gwMAC, gwLinkIdx, err = util.SetLinkUp(i.hostGateway) + gwLinkIdx, err = setLinkUp(i.hostGateway) if err == nil { break } @@ -696,8 +703,7 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int return err } - i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC, OFPort: uint32(gatewayIface.OFPort)} - gatewayIface.MAC = gwMAC + i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gatewayIface.MAC, OFPort: uint32(gatewayIface.OFPort)} gatewayIface.IPs = []net.IP{} if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { // Assign IP to gw as required by SpoofGuard. @@ -1146,13 +1152,13 @@ func (i *Initializer) allocateGatewayAddresses(localSubnets []*net.IPNet, gatewa // (i.e. portExists is false). Indeed, it may be possible for the interface to exist even if the OVS bridge does // not exist. // Configure any missing IP address on the interface. Remove any extra IP address that may exist. - if err := util.ConfigureLinkAddresses(i.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil { + if err := configureLinkAddresses(i.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil { return err } // Periodically check whether IP configuration of the gateway is correct. // Terminate when stopCh is closed. go wait.Until(func() { - if err := util.ConfigureLinkAddresses(i.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil { + if err := configureLinkAddresses(i.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil { klog.Errorf("Failed to check IP configuration of the gateway: %v", err) } }, 60*time.Second, i.stopCh) diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 790e41f195d..57acefff588 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -235,7 +235,7 @@ func (i *Initializer) ConnectUplinkToOVSBridge() error { if err != nil { return err } - if _, _, err = util.SetLinkUp(uplinkName); err != nil { + if _, err = util.SetLinkUp(uplinkName); err != nil { return err } if err = util.ConfigureLinkAddresses(localLink.Attrs().Index, uplinkIPs); err != nil { diff --git a/pkg/agent/agent_linux_test.go b/pkg/agent/agent_linux_test.go new file mode 100644 index 00000000000..5abb5cc4e90 --- /dev/null +++ b/pkg/agent/agent_linux_test.go @@ -0,0 +1,19 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +func mockSetInterfaceMTU(returnErr error) func() { + return func() {} +} diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 7e4dd7dfce1..c1ae04755d1 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -578,3 +578,70 @@ func TestSetupDefaultTunnelInterface(t *testing.T) { }) } } + +func TestSetupGatewayInterface(t *testing.T) { + defer mockSetLinkUp(10, nil)() + defer mockConfigureLinkAddress(nil)() + defer mockSetInterfaceMTU(nil)() + + controller := mock.NewController(t) + defer controller.Finish() + + podCIDRStr := "172.16.10.0/24" + _, podCIDR, _ := net.ParseCIDR(podCIDRStr) + nodeConfig := &config.NodeConfig{ + Name: "n1", + Type: config.K8sNode, + OVSBridge: "br-int", + PodIPv4CIDR: podCIDR, + NodeMTU: 1450, + } + networkConfig := &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + TunnelType: ovsconfig.GeneveTunnel, + TunnelCsum: false, + } + + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(controller) + client := fake.NewSimpleClientset() + ifaceStore := interfacestore.NewInterfaceStore() + stopCh := make(chan struct{}) + initializer := &Initializer{ + client: client, + ifaceStore: ifaceStore, + ovsBridgeClient: mockOVSBridgeClient, + ovsBridge: "br-int", + networkConfig: networkConfig, + nodeConfig: nodeConfig, + hostGateway: "antrea-gw0", + stopCh: stopCh, + } + close(stopCh) + portUUID := "123456780a" + ofport := int32(config.HostGatewayOFPort) + mockOVSBridgeClient.EXPECT().CreateInternalPort(initializer.hostGateway, ofport, mock.Any(), mock.Any()).Return(portUUID, nil) + mockOVSBridgeClient.EXPECT().GetOFPort(initializer.hostGateway, false).Return(ofport, nil) + mockOVSBridgeClient.EXPECT().SetInterfaceMTU(initializer.hostGateway, nodeConfig.NodeMTU).Return(nil) + err := initializer.setupGatewayInterface() + assert.NoError(t, err) +} + +func mockSetLinkUp(returnIndex int, returnErr error) func() { + originalSetLinkUp := setLinkUp + setLinkUp = func(name string) (int, error) { + return returnIndex, returnErr + } + return func() { + setLinkUp = originalSetLinkUp + } +} + +func mockConfigureLinkAddress(returnedErr error) func() { + originalConfigureLinkAddresses := configureLinkAddresses + configureLinkAddresses = func(idx int, ipNets []*net.IPNet) error { + return returnedErr + } + return func() { + configureLinkAddresses = originalConfigureLinkAddresses + } +} diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index df7f17b2b28..104b53786e1 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -34,6 +34,11 @@ import ( utilip "antrea.io/antrea/pkg/util/ip" ) +var ( + // setInterfaceMTU is meant to be overridden for testing + setInterfaceMTU = util.SetInterfaceMTU +) + func (i *Initializer) prepareHostNetwork() error { if i.nodeConfig.Type == config.K8sNode { return i.prepareHNSNetworkAndOVSExtension() @@ -419,7 +424,7 @@ func (i *Initializer) setInterfaceMTU(iface string, mtu int) error { if err := i.ovsBridgeClient.SetInterfaceMTU(iface, mtu); err != nil { return err } - return util.SetInterfaceMTU(iface, mtu) + return setInterfaceMTU(iface, mtu) } func (i *Initializer) setVMNodeConfig(en *v1alpha1.ExternalNode, nodeName string) error { diff --git a/pkg/agent/agent_windows_test.go b/pkg/agent/agent_windows_test.go new file mode 100644 index 00000000000..97c04f33c16 --- /dev/null +++ b/pkg/agent/agent_windows_test.go @@ -0,0 +1,25 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +func mockSetInterfaceMTU(returnErr error) func() { + originalSetInterfaceMTU := setInterfaceMTU + setInterfaceMTU = func(ifaceName string, mtu int) error { + return returnErr + } + return func() { + setInterfaceMTU = originalSetInterfaceMTU + } +} diff --git a/pkg/agent/cniserver/interface_configuration_linux.go b/pkg/agent/cniserver/interface_configuration_linux.go index eced5fe74ae..a45264ac917 100644 --- a/pkg/agent/cniserver/interface_configuration_linux.go +++ b/pkg/agent/cniserver/interface_configuration_linux.go @@ -37,6 +37,7 @@ import ( "antrea.io/antrea/pkg/agent/util/ndp" cnipb "antrea.io/antrea/pkg/apis/cni/v1beta1" "antrea.io/antrea/pkg/ovs/ovsconfig" + cniip "antrea.io/antrea/third_party/containernetworking/ip" ) // NetDeviceType type Enum @@ -254,13 +255,14 @@ func (ic *ifConfigurator) configureContainerLinkVeth( containerIface := ¤t.Interface{Name: containerIfaceName, Sandbox: containerNetNS} result.Interfaces = []*current.Interface{hostIface, containerIface} + podMAC := util.GenerateRandomMAC() if err := ns.WithNetNSPath(containerNetNS, func(hostNS ns.NetNS) error { klog.V(2).Infof("Creating veth devices (%s, %s) for container %s", containerIfaceName, hostIfaceName, containerID) - hostVeth, containerVeth, err := ip.SetupVethWithName(containerIfaceName, hostIfaceName, mtu, hostNS) + hostVeth, containerVeth, err := cniip.SetupVethWithName(containerIfaceName, hostIfaceName, mtu, podMAC.String(), hostNS) if err != nil { return fmt.Errorf("failed to create veth devices for container %s: %v", containerID, err) } - containerIface.Mac = containerVeth.HardwareAddr.String() + containerIface.Mac = podMAC.String() hostIface.Mac = hostVeth.HardwareAddr.String() // Disable TX checksum offloading when it's configured explicitly. if ic.disableTXChecksumOffload { diff --git a/pkg/agent/controller/trafficcontrol/controller.go b/pkg/agent/controller/trafficcontrol/controller.go index 3ed0675184e..bc91c31a558 100644 --- a/pkg/agent/controller/trafficcontrol/controller.go +++ b/pkg/agent/controller/trafficcontrol/controller.go @@ -590,7 +590,7 @@ func (c *Controller) createOVSInternalPort(portName string) (string, error) { return "", err } if pollErr := wait.PollImmediate(time.Second, 5*time.Second, func() (bool, error) { - _, _, err := util.SetLinkUp(portName) + _, err := util.SetLinkUp(portName) if err == nil { return true, nil } diff --git a/pkg/agent/interfacestore/interface_cache_test.go b/pkg/agent/interfacestore/interface_cache_test.go index 37ec02e7d22..de41c9a597e 100644 --- a/pkg/agent/interfacestore/interface_cache_test.go +++ b/pkg/agent/interfacestore/interface_cache_test.go @@ -89,7 +89,7 @@ func testContainerInterface(t *testing.T) { } func testGatewayInterface(t *testing.T) { - gatewayInterface := NewGatewayInterface("antrea-gw0") + gatewayInterface := NewGatewayInterface("antrea-gw0", util.GenerateRandomMAC()) gatewayInterface.IPs = []net.IP{gwIP} gatewayInterface.OVSPortConfig = &OVSPortConfig{ OFPort: 13, diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index 903af7169ee..b0e03a31b35 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -146,8 +146,8 @@ func NewContainerInterface( } // NewGatewayInterface creates InterfaceConfig for the host gateway interface. -func NewGatewayInterface(gatewayName string) *InterfaceConfig { - gatewayConfig := &InterfaceConfig{InterfaceName: gatewayName, Type: GatewayInterface} +func NewGatewayInterface(gatewayName string, gatewayMAC net.HardwareAddr) *InterfaceConfig { + gatewayConfig := &InterfaceConfig{InterfaceName: gatewayName, Type: GatewayInterface, MAC: gatewayMAC} return gatewayConfig } diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 33fdc487dab..46c254d0365 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -406,8 +406,9 @@ func GenerateRandomMAC() net.HardwareAddr { if _, err := rand.Read(buf); err != nil { klog.ErrorS(err, "Failed to generate a random MAC") } - // Set the local bit - buf[0] |= 2 + // Unset the multicast bit. + buf[0] &= 0xfe + buf[0] |= 0x02 return buf } diff --git a/pkg/agent/util/net_linux.go b/pkg/agent/util/net_linux.go index 3a9a14111a8..fb6374e742a 100644 --- a/pkg/agent/util/net_linux.go +++ b/pkg/agent/util/net_linux.go @@ -106,22 +106,21 @@ func GetNSPath(netnsName string) (string, error) { return netNS.Path(), nil } -func SetLinkUp(name string) (net.HardwareAddr, int, error) { +func SetLinkUp(name string) (int, error) { link, err := netlink.LinkByName(name) if err != nil { if _, ok := err.(netlink.LinkNotFoundError); ok { - return nil, 0, newLinkNotFoundError(name) + return 0, newLinkNotFoundError(name) } - return nil, 0, err + return 0, err } // Set host gateway interface up. if err := netlink.LinkSetUp(link); err != nil { klog.Errorf("Failed to set host link for %s up: %v", name, err) - return nil, 0, err + return 0, err } - mac := link.Attrs().HardwareAddr index := link.Attrs().Index - return mac, index, nil + return index, nil } func addrSliceDifference(s1, s2 []netlink.Addr) []*netlink.Addr { diff --git a/pkg/agent/util/net_test.go b/pkg/agent/util/net_test.go index 30b4528630e..f58a8f1a4b9 100644 --- a/pkg/agent/util/net_test.go +++ b/pkg/agent/util/net_test.go @@ -104,3 +104,15 @@ func TestExtendCIDRWithIP(t *testing.T) { assert.Equal(t, expectedIPNet, gotIPNet) } } + +func TestGenerateRandomMAC(t *testing.T) { + validateBits := func(mac net.HardwareAddr) (byte, byte) { + localBit := mac[0] & 0x2 >> 1 + mcastBit := mac[0] & 0x1 + return localBit, mcastBit + } + mac1 := GenerateRandomMAC() + localBit, mcastBit := validateBits(mac1) + assert.Equal(t, uint8(1), localBit) + assert.Equal(t, uint8(0), mcastBit) +} diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index 36435143b8d..921e272b7e2 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -289,26 +289,25 @@ func EnableHNSNetworkExtension(hnsNetID string, vSwitchExtension string) error { return nil } -func SetLinkUp(name string) (net.HardwareAddr, int, error) { +func SetLinkUp(name string) (int, error) { // Set host gateway interface up. if err := EnableHostInterface(name); err != nil { klog.Errorf("Failed to set host link for %s up: %v", name, err) if strings.Contains(err.Error(), "ObjectNotFound") { - return nil, 0, newLinkNotFoundError(name) + return 0, newLinkNotFoundError(name) } - return nil, 0, err + return 0, err } iface, err := net.InterfaceByName(name) if err != nil { if opErr, ok := err.(*net.OpError); ok && opErr.Err.Error() == "no such network interface" { - return nil, 0, newLinkNotFoundError(name) + return 0, newLinkNotFoundError(name) } - return nil, 0, err + return 0, err } - mac := iface.HardwareAddr index := iface.Index - return mac, index, nil + return index, nil } func addrEqual(addr1, addr2 *net.IPNet) bool { diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index bcc1d7c9ffe..910ca0ce7a2 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -48,6 +48,7 @@ type OVSPortData struct { OFPort int32 ExternalIDs map[string]string Options map[string]string + MAC net.HardwareAddr } const ( @@ -694,6 +695,19 @@ func buildPortDataCommon(port, intf map[string]interface{}, portData *OVSPortDat } else { // ofport not assigned by OVS yet portData.OFPort = 0 } + var macInUse string + if field, ok := intf["mac_in_use"].(string); ok { + macInUse = field + } else if fields, ok := intf["mac_in_use"].([]interface{}); ok { + if len(fields) > 0 { + macInUse = fields[0].(string) + } + } + if macInUse != "" { + if mac, err := net.ParseMAC(macInUse); err == nil { + portData.MAC = mac + } + } } // GetPortData retrieves port data given the OVS port UUID and interface name. @@ -709,7 +723,7 @@ func (br *OVSBridge) GetPortData(portUUID, ifName string) (*OVSPortData, Error) }) tx.Select(dbtransaction.Select{ Table: "Interface", - Columns: []string{"_uuid", "type", "ofport", "options"}, + Columns: []string{"_uuid", "type", "ofport", "options", "mac_in_use"}, Where: [][]interface{}{{"name", "==", ifName}}, }) @@ -762,7 +776,7 @@ func (br *OVSBridge) GetPortList() ([]OVSPortData, Error) { }) tx.Select(dbtransaction.Select{ Table: "Interface", - Columns: []string{"_uuid", "type", "name", "ofport", "options"}, + Columns: []string{"_uuid", "type", "name", "ofport", "options", "mac_in_use"}, }) res, err, temporary := tx.Commit() diff --git a/pkg/ovs/ovsconfig/ovs_client_test.go b/pkg/ovs/ovsconfig/ovs_client_test.go index f8e24a1a00e..7b47a95da11 100644 --- a/pkg/ovs/ovsconfig/ovs_client_test.go +++ b/pkg/ovs/ovsconfig/ovs_client_test.go @@ -15,6 +15,7 @@ package ovsconfig import ( + "net" "testing" "github.com/stretchr/testify/assert" @@ -37,3 +38,83 @@ func TestOVSClient(t *testing.T) { assert.NoError(t, err) } + +func TestBuildPortDataCommon(t *testing.T) { + macStr := "9a:23:45:23:22:41" + intfMAC, _ := net.ParseMAC(macStr) + for _, tc := range []struct { + name string + port map[string]interface{} + intf map[string]interface{} + portData *OVSPortData + }{ + { + name: "gw-port", + port: map[string]interface{}{"name": "antrea-gw0", "external_ids": []interface{}{"map", []interface{}{[]interface{}{"antrea-type", "gateway"}}}}, + intf: map[string]interface{}{"name": "antrea-gw0", "mac_in_use": macStr, "type": "internal", "ofport": float64(2), "options": []interface{}{""}}, + portData: &OVSPortData{ + Name: "antrea-gw0", + ExternalIDs: map[string]string{"antrea-type": "gateway"}, + Options: map[string]string{}, + IFType: "internal", + OFPort: 2, + MAC: intfMAC, + }, + }, { + name: "tun-port", + port: map[string]interface{}{"name": "antrea-tun0", "external_ids": []interface{}{"map", []interface{}{[]interface{}{"antrea-type", "tunnel"}}}}, + intf: map[string]interface{}{"name": "antrea-tun0", "mac_in_use": macStr, "type": "geneve", "ofport": float64(1), "options": []interface{}{"map", []interface{}{[]interface{}{"key", "flow"}, []interface{}{"remote_ip", "flow"}}}}, + portData: &OVSPortData{ + Name: "antrea-tun0", + ExternalIDs: map[string]string{"antrea-type": "tunnel"}, + Options: map[string]string{"key": "flow", "remote_ip": "flow"}, + IFType: "geneve", + OFPort: 1, + MAC: intfMAC, + }, + }, { + name: "general-port", + port: map[string]interface{}{"name": "p0", "external_ids": []interface{}{"map", []interface{}{[]interface{}{"antrea-type", "container"}, []interface{}{"ip", "1.2.3.4"}}}}, + intf: map[string]interface{}{"name": "p0", "mac_in_use": []interface{}{macStr}, "type": "", "ofport": float64(3), "options": []interface{}{""}}, + portData: &OVSPortData{ + Name: "p0", + ExternalIDs: map[string]string{"antrea-type": "container", "ip": "1.2.3.4"}, + Options: map[string]string{}, + IFType: "", + OFPort: 3, + MAC: intfMAC, + }, + }, { + name: "access-port", + port: map[string]interface{}{"name": "p1", "tag": float64(10), "external_ids": []interface{}{"map", []interface{}{[]interface{}{"antrea-type", "container"}, []interface{}{"ip", "1.2.3.5"}}}}, + intf: map[string]interface{}{"name": "p1", "mac_in_use": macStr, "type": "", "ofport": float64(3), "options": []interface{}{""}}, + portData: &OVSPortData{ + Name: "p1", + ExternalIDs: map[string]string{"antrea-type": "container", "ip": "1.2.3.5"}, + Options: map[string]string{}, + IFType: "", + OFPort: 3, + VLANID: 10, + MAC: intfMAC, + }, + }, { + name: "no-mac-port", + port: map[string]interface{}{"name": "p2", "external_ids": []interface{}{"map", []interface{}{[]interface{}{"antrea-type", "container"}, []interface{}{"ip", "1.2.3.5"}}}}, + intf: map[string]interface{}{"name": "p2", "mac_in_use": []interface{}{}, "type": "", "ofport": float64(4), "options": []interface{}{""}}, + portData: &OVSPortData{ + Name: "p2", + ExternalIDs: map[string]string{"antrea-type": "container", "ip": "1.2.3.5"}, + Options: map[string]string{}, + IFType: "", + OFPort: 4, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + portData := &OVSPortData{} + buildPortDataCommon(tc.port, tc.intf, portData) + assert.Equal(t, tc.portData, portData) + }) + } + +} diff --git a/test/integration/agent/net_linux_test.go b/test/integration/agent/net_linux_test.go index f405e5b164f..62059da957d 100644 --- a/test/integration/agent/net_linux_test.go +++ b/test/integration/agent/net_linux_test.go @@ -36,7 +36,7 @@ func createTestInterface(t *testing.T, name string) string { } func setTestInterfaceUp(t *testing.T, name string) int { - _, ifaceIdx, err := util.SetLinkUp(name) + ifaceIdx, err := util.SetLinkUp(name) require.NoError(t, err) return ifaceIdx } diff --git a/test/integration/agent/net_windows_test.go b/test/integration/agent/net_windows_test.go index 3b04a3f796d..cee60dc3577 100644 --- a/test/integration/agent/net_windows_test.go +++ b/test/integration/agent/net_windows_test.go @@ -66,7 +66,7 @@ func createTestInterface(t *testing.T, name string) string { } func setTestInterfaceUp(t *testing.T, name string) int { - _, ifaceIdx, err := util.SetLinkUp(adapterName(name)) + ifaceIdx, err := util.SetLinkUp(adapterName(name)) require.NoError(t, err) return ifaceIdx } diff --git a/third_party/containernetworking/ip/link_linux.go b/third_party/containernetworking/ip/link_linux.go new file mode 100644 index 00000000000..12b35100763 --- /dev/null +++ b/third_party/containernetworking/ip/link_linux.go @@ -0,0 +1,152 @@ +// Copyright 2015 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ip + +import ( + "crypto/rand" + "fmt" + "net" + "os" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/utils/sysctl" + "github.com/vishvananda/netlink" +) + +// makeVethPair is called from within the container's network namespace +func makeVethPair(name, peer string, mtu int, mac string, hostNS ns.NetNS) (netlink.Link, error) { + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: name, + MTU: mtu, + }, + PeerName: peer, + PeerNamespace: netlink.NsFd(int(hostNS.Fd())), + } + if mac != "" { + m, err := net.ParseMAC(mac) + if err != nil { + return nil, err + } + veth.LinkAttrs.HardwareAddr = m + } + if err := netlink.LinkAdd(veth); err != nil { + return nil, err + } + // Re-fetch the container link to get its creation-time parameters, e.g. index and mac + veth2, err := netlink.LinkByName(name) + if err != nil { + netlink.LinkDel(veth) // try and clean up the link if possible. + return nil, err + } + + return veth2, nil +} + +func peerExists(name string) bool { + if _, err := netlink.LinkByName(name); err != nil { + return false + } + return true +} + +func makeVeth(name, vethPeerName string, mtu int, mac string, hostNS ns.NetNS) (peerName string, veth netlink.Link, err error) { + for i := 0; i < 10; i++ { + if vethPeerName != "" { + peerName = vethPeerName + } else { + peerName, err = RandomVethName() + if err != nil { + return + } + } + + veth, err = makeVethPair(name, peerName, mtu, mac, hostNS) + switch { + case err == nil: + return + + case os.IsExist(err): + if peerExists(peerName) && vethPeerName == "" { + continue + } + err = fmt.Errorf("container veth name provided (%v) already exists", name) + return + + default: + err = fmt.Errorf("failed to make veth pair: %v", err) + return + } + } + + // should really never be hit + err = fmt.Errorf("failed to find a unique veth name") + return +} + +// RandomVethName returns string "veth" with random prefix (hashed from entropy) +func RandomVethName() (string, error) { + entropy := make([]byte, 4) + _, err := rand.Read(entropy) + if err != nil { + return "", fmt.Errorf("failed to generate random veth name: %v", err) + } + + // NetworkManager (recent versions) will ignore veth devices that start with "veth" + return fmt.Sprintf("veth%x", entropy), nil +} + +func ifaceFromNetlinkLink(l netlink.Link) net.Interface { + a := l.Attrs() + return net.Interface{ + Index: a.Index, + MTU: a.MTU, + Name: a.Name, + HardwareAddr: a.HardwareAddr, + Flags: a.Flags, + } +} + +// SetupVethWithName sets up a pair of virtual ethernet devices. +// Call SetupVethWithName from inside the container netns. It will create both veth +// devices and move the host-side veth into the provided hostNS namespace. +// hostVethName: If hostVethName is not specified, the host-side veth name will use a random string. +// On success, SetupVethWithName returns (hostVeth, containerVeth, nil) +func SetupVethWithName(contVethName, hostVethName string, mtu int, contVethMac string, hostNS ns.NetNS) (net.Interface, net.Interface, error) { + hostVethName, contVeth, err := makeVeth(contVethName, hostVethName, mtu, contVethMac, hostNS) + if err != nil { + return net.Interface{}, net.Interface{}, err + } + + var hostVeth netlink.Link + err = hostNS.Do(func(_ ns.NetNS) error { + hostVeth, err = netlink.LinkByName(hostVethName) + if err != nil { + return fmt.Errorf("failed to lookup %q in %q: %v", hostVethName, hostNS.Path(), err) + } + + if err = netlink.LinkSetUp(hostVeth); err != nil { + return fmt.Errorf("failed to set %q up: %v", hostVethName, err) + } + + // we want to own the routes for this interface + _, _ = sysctl.Sysctl(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0") + return nil + }) + if err != nil { + return net.Interface{}, net.Interface{}, err + } + return ifaceFromNetlinkLink(hostVeth), ifaceFromNetlinkLink(contVeth), nil +}