Merge pull request #48 from naserdean/add_SF_support

Add SF support

sriovnet.go

@@ -24,7 +24,10 @@ const (
 	ibEncapType    = "infiniband"
 )
 
-var virtFnRe = regexp.MustCompile(`virtfn(\d+)`)
+var (
+	virtFnRe     = regexp.MustCompile(`virtfn(\d+)`)
+	pciAddressRe = regexp.MustCompile(`^[0-9a-f]{4}:[0-9a-f]{2}:[01][0-9a-f].[0-7]$`)
+)
 
 type VfObj struct {
 	Index      int
@@ -429,27 +432,6 @@ func GetVfIndexByPciAddress(vfPciAddress string) (int, error) {
 	return -1, fmt.Errorf("vf index for %s not found", vfPciAddress)
 }
 
-// GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and
-// returns the correlate list of netdevices
-func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
-	pciDir := filepath.Join(PciSysDir, pciAddress, "net")
-	_, err := utilfs.Fs.Stat(pciDir)
-	if err != nil {
-		return nil, fmt.Errorf("cannot get a network device with pci address %v %v", pciAddress, err)
-	}
-
-	netDevicesFiles, err := utilfs.Fs.ReadDir(pciDir)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get network device name in %v %v", pciDir, err)
-	}
-
-	netDevices := make([]string, 0, len(netDevicesFiles))
-	for _, netDeviceFile := range netDevicesFiles {
-		netDevices = append(netDevices, strings.TrimSpace(netDeviceFile.Name()))
-	}
-	return netDevices, nil
-}
-
 // GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format
 func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
 	pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn")
@@ -464,3 +446,10 @@ func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
 	}
 	return pf, err
 }
+
+// GetNetDevicesFromPci gets a PCI address (e.g '0000:03:00.1') and
+// returns the correlate list of netdevices
+func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
+	pciDir := filepath.Join(PciSysDir, pciAddress, "net")
+	return getFileNamesFromPath(pciDir)
+}

sriovnet_aux.go

@@ -0,0 +1,88 @@
+/*----------------------------------------------------
+ *
+ *  2022 NVIDIA CORPORATION & AFFILIATES
+ *
+ *  Licensed under the Apache License, Version 2.0 (the License);
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an AS IS BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *----------------------------------------------------
+ */
+
+package sriovnet
+
+import (
+	"fmt"
+	"path/filepath"
+	"strconv"
+
+	utilfs "github.com/Mellanox/sriovnet/pkg/utils/filesystem"
+)
+
+// GetNetDeviceFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and
+// returns the correlate netdevice
+func GetNetDevicesFromAux(auxDev string) ([]string, error) {
+	auxDir := filepath.Join(AuxSysDir, auxDev, "net")
+	return getFileNamesFromPath(auxDir)
+}
+
+// GetSfIndexByAuxDev gets a SF device name (e.g 'mlx5_core.sf.2') and
+// returns the correlate SF index.
+func GetSfIndexByAuxDev(auxDev string) (int, error) {
+	sfNumFile := filepath.Join(AuxSysDir, auxDev, "sfnum")
+	if _, err := utilfs.Fs.Stat(sfNumFile); err != nil {
+		return -1, fmt.Errorf("cannot get sfnum for %s device: %v", auxDev, err)
+	}
+
+	sfNumStr, err := utilfs.Fs.ReadFile(sfNumFile)
+	if err != nil {
+		return -1, fmt.Errorf("cannot read sfnum file for %s device: %v", auxDev, err)
+	}
+
+	sfnum, err := strconv.Atoi(string(sfNumStr))
+	if err != nil {
+		return -1, err
+	}
+	return sfnum, nil
+}
+
+// GetPfPciFromAux retrieves the parent PF PCI address of the provided auxiliary device in D.T.f format
+func GetPfPciFromAux(auxDev string) (string, error) {
+	auxPath := filepath.Join(AuxSysDir, auxDev)
+	absoluteAuxPath, err := utilfs.Fs.Readlink(auxPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read auxiliary link, provided device ID may be not auxiliary device. %v", err)
+	}
+	// /sys/bus/auxiliary/devices/mlx5_core.sf.7 ->
+	//		./../../devices/pci0000:00/0000:00:00.0/0000:01:00.0/0000:02:00.0/0000:03:00.0/mlx5_core.sf.7
+	parent := filepath.Dir(absoluteAuxPath)
+	base := filepath.Base(parent)
+	for !pciAddressRe.MatchString(base) {
+		// it's a nested auxiliary device. repeat
+		parent = filepath.Dir(parent)
+		base = filepath.Base(parent)
+	}
+	if base == "" {
+		return base, fmt.Errorf("could not find PF PCI Address")
+	}
+	return base, err
+}
+
+// GetUplinkRepresentorFromAux gets auxiliary device name (e.g 'mlx5_core.sf.2') and
+// returns the uplink representor netdev name for device.
+func GetUplinkRepresentorFromAux(auxDev string) (string, error) {
+	pfPci, err := GetPfPciFromAux(auxDev)
+	if err != nil {
+		return "", fmt.Errorf("failed to find uplink PCI device: %v", err)
+	}
+
+	return GetUplinkRepresentor(pfPci)
+}

sriovnet_aux_test.go

@@ -0,0 +1,187 @@
+/*----------------------------------------------------
+ *
+ *  2022 NVIDIA CORPORATION & AFFILIATES
+ *
+ *  Licensed under the Apache License, Version 2.0 (the License);
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an AS IS BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ *----------------------------------------------------
+ */
+
+package sriovnet
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	utilfs "github.com/Mellanox/sriovnet/pkg/utils/filesystem"
+)
+
+func TestGetNetDevicesFromAuxSuccess(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+	netDevName := "en3f0pf0sf0"
+	path := filepath.Join(AuxSysDir, auxDevName, "net", netDevName)
+	_ = utilfs.Fs.MkdirAll(path, os.FileMode(0755))
+
+	devNames, err := GetNetDevicesFromAux(auxDevName)
+	assert.NoError(t, err)
+	assert.Equal(t, netDevName, devNames[0])
+}
+
+func TestGetNetDevicesFromAuxErrorNoAux(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+	devNames, err := GetNetDevicesFromAux(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, ([]string(nil)), devNames)
+}
+
+func TestGetNetDevicesFromAuxErrorNoDevice(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+
+	_ = utilfs.Fs.MkdirAll(filepath.Join(AuxSysDir, auxDevName, "net"), os.FileMode(0755))
+	devNames, _ := GetNetDevicesFromAux(auxDevName)
+	assert.Equal(t, ([]string{}), devNames)
+}
+
+func TestGetSfIndexByAuxDevSuccess(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+	auxDevPath := filepath.Join(AuxSysDir, auxDevName)
+	sfNumFile := filepath.Join(auxDevPath, "sfnum")
+
+	_ = utilfs.Fs.MkdirAll(auxDevPath, os.FileMode(0755))
+	_ = utilfs.Fs.WriteFile(sfNumFile, []byte("0"), os.FileMode(0644))
+
+	sfIndex, err := GetSfIndexByAuxDev(auxDevName)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, sfIndex)
+}
+
+func TestGetSfIndexByAuxDevErrorNoSfNum(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+	expectedError := "cannot get sfnum"
+
+	sfIndex, err := GetSfIndexByAuxDev(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, -1, sfIndex)
+	assert.Contains(t, err.Error(), expectedError)
+}
+
+func TestGetSfIndexByAuxDevErrorRead(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.19"
+	auxDevPath := filepath.Join(AuxSysDir, auxDevName)
+	sfNumFile := filepath.Join(auxDevPath, "sfnum1")
+	expectedError := "cannot get sfnum"
+
+	_ = utilfs.Fs.MkdirAll(auxDevPath, os.FileMode(0755))
+	_ = utilfs.Fs.WriteFile(sfNumFile, []byte("0"), os.FileMode(0))
+
+	sfIndex, err := GetSfIndexByAuxDev(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, -1, sfIndex)
+	assert.Contains(t, err.Error(), expectedError)
+}
+
+func TestGetSfIndexByAuxDevErrorAtoi(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	auxDevName := "mlx5_core.sf.0"
+	auxDevPath := filepath.Join(AuxSysDir, auxDevName)
+	sfNumFile := filepath.Join(auxDevPath, "sfnum")
+	expectedError := "invalid syntax"
+
+	_ = utilfs.Fs.MkdirAll(auxDevPath, os.FileMode(0755))
+	_ = utilfs.Fs.WriteFile(sfNumFile, []byte("NaN"), os.FileMode(0644))
+
+	sfIndex, err := GetSfIndexByAuxDev(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, -1, sfIndex)
+	assert.Contains(t, err.Error(), expectedError)
+}
+
+func TestGetPfPciFromAux(t *testing.T) {
+	teardown := setupFakeFs(t)
+	defer teardown()
+	// Create PCI & Auxiliary sysfs layout with FakeFs
+	pfPciAddr := "0000:02:00.0"
+	auxDevName := "mlx5_core.eth.0"
+	auxDevPath := filepath.Join(PciSysDir, pfPciAddr, auxDevName)
+	auxDevLink := filepath.Join(AuxSysDir, auxDevName)
+
+	// PF PCI path and auxiliary device dir
+	_ = utilfs.Fs.MkdirAll(auxDevPath, os.FileMode(0755))
+	_ = utilfs.Fs.MkdirAll(AuxSysDir, os.FileMode(0755))
+	// Auxiliary device link
+	_ = utilfs.Fs.Symlink(auxDevPath, auxDevLink)
+
+	pf, err := GetPfPciFromAux(auxDevName)
+	assert.NoError(t, err)
+	assert.Equal(t, pfPciAddr, pf)
+}
+
+func TestGetPfPciFromAuxNoSuchDevice(t *testing.T) {
+	// Create PCI sysfs layout with FakeFs
+	auxDevName := "mlx5_core.eth.0"
+
+	pf, err := GetPfPciFromAux(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, "", pf)
+}
+
+func TestGetUplinkRepresentorFromAux(t *testing.T) {
+	teardownFs := setupFakeFs(t)
+	defer teardownFs()
+	// Create PCI & Auxiliary sysfs layout with FakeFs
+	pfPciAddr := "0000:02:00.0"
+	auxDevName := "mlx5_core.eth.0"
+	auxDevPath := filepath.Join(PciSysDir, pfPciAddr, auxDevName)
+	auxDevLink := filepath.Join(AuxSysDir, auxDevName)
+
+	uplinkRep := &repContext{"eth0", "p0", "111111"}
+	sfsReps := []*repContext{{"enp_0", "pf0sf0", "0123"}}
+
+	teardownUplink := setupUplinkRepresentorEnv(t, uplinkRep, pfPciAddr, sfsReps)
+	defer teardownUplink()
+
+	// PF PCI path and auxiliary device dir
+	_ = utilfs.Fs.MkdirAll(auxDevPath, os.FileMode(0755))
+	_ = utilfs.Fs.MkdirAll(AuxSysDir, os.FileMode(0755))
+	// Auxiliary device link
+	_ = utilfs.Fs.Symlink(auxDevPath, auxDevLink)
+
+	pf, err := GetUplinkRepresentorFromAux(auxDevName)
+	assert.NoError(t, err)
+	assert.Equal(t, "eth0", pf)
+}
+
+func TestGetUplinkRepresentorFromAuxNoSuchDevice(t *testing.T) {
+	// Create PCI sysfs layout with FakeFs
+	auxDevName := "mlx5_core.eth.0"
+
+	pf, err := GetUplinkRepresentorFromAux(auxDevName)
+	assert.Error(t, err)
+	assert.Equal(t, "", pf)
+}

sriovnet_helper.go

@@ -10,6 +10,7 @@ import (
 const (
 	NetSysDir        = "/sys/class/net"
 	PciSysDir        = "/sys/bus/pci/devices"
+	AuxSysDir        = "/sys/bus/auxiliary/devices"
 	pcidevPrefix     = "device"
 	netdevDriverDir  = "device/driver"
 	netdevUnbindFile = "unbind"

sriovnet_integration_test.go

@@ -325,6 +325,36 @@ func TestIntegrationGetVfRepresentor(t *testing.T) {
 	}
 }
 
+func TestIntegrationGetSfRepresentor(t *testing.T) {
+	tcases := []struct {
+		uplink     string
+		sfNum      int
+		expected   string
+		shouldFail bool
+	}{
+		{uplink: "p0", sfNum: 2, expected: "en3f0pf0sf2", shouldFail: false},
+		{uplink: "foobar", sfNum: 2, expected: "", shouldFail: true},
+		{uplink: "enp3s0", sfNum: 44, expected: "", shouldFail: true},
+	}
+
+	for _, tcase := range tcases {
+		sfRep, err := GetSfRepresentor(tcase.uplink, tcase.sfNum)
+		if tcase.shouldFail {
+			if err == nil {
+				t.Fatal("Expected failure but no error occured")
+			}
+			continue
+		}
+		if err != nil {
+			t.Fatal("GetSfRepresentor failed with error: ", err)
+		}
+		if sfRep != tcase.expected {
+			t.Fatal("Actual Representor does not match expected Representor", sfRep, "!=", tcase.expected)
+		}
+		t.Log("GetSfRepresentor", "uplink: ", tcase.uplink, " SF Index: ", tcase.sfNum, " Rep: ", sfRep)
+	}
+}
+
 func TestIntegrationGetUplinkRepresentor(t *testing.T) {
 	uplink := "enp3s0f0np0"
 	pfPciAddress := "0000:03:00.0"
@@ -340,7 +370,7 @@ func TestIntegrationGetUplinkRepresentor(t *testing.T) {
 		t.Fatal("Actual Representor does not match expected Representor", rep, "!=", uplink)
 	}
 
-	rep, err := GetUplinkRepresentor(vfPciAddress)
+	rep, err = GetUplinkRepresentor(vfPciAddress)
 
 	if err != nil {
 		t.Fatal("GetUplinkRepresentor failed with error: ", err)