rootBusPath: create rootBusPath dynamically.

Currently, rootBusPath is set as a constant value. Bus this pcie bus path
is not alway static, eg. the rootBus on arm64 is
"/devices/platform/4010000000.pcie/pci0000:00". The address part of
"4010000000.pcie" may vary with the maxMem in qemu so it should not be a
fixed value.

For exmaple:
HIGH PCIE address reserved in qemu on virt is from 0 to 512G. The lower
part of address may be allocated to normal memory. In general, normal memory
is largely less than 256G, so the base address of HIGH PCIE can be 256G.
But, in case the maxmem is large enough, like around 256G, the base address
of HIGH PCIE must be increase, eg. 300G.
In the current implementation of kata-runtime, the maxmem in qemu is the host
memory size. So if the host memory size is large enough, the prefix of the
pci device path name will be different from rootBusPath set in kata-agent which
can lead to failure on device hotplug.

This patch offer a mechanism to create rootBusPath dynamically but only give
implemention for arm64 and return the default value of rootBusPath for other arch.

Fixes: #859
Signed-off-by: Jianyong Wu <[email protected]>

agent.go

@@ -709,6 +709,11 @@ func (s *sandbox) waitForStopServer() {
 
 func (s *sandbox) listenToUdevEvents() {
 	fieldLogger := agentLog.WithField("subsystem", "udevlistener")
+	rootBusPath, err := createRootBusPath()
+	if err != nil {
+		fieldLogger.Warnf("Error creating root bus path")
+		return
+	}
 
 	uEvHandler, err := uevent.NewHandler()
 	if err != nil {

device.go

@@ -109,6 +109,10 @@ func rescanPciBus() error {
 func pciPathToSysfsImpl(pciPath PciPath) (string, error) {
 	var relPath string
 	bus := "0000:00"
+	rootBusPath, err := createRootBusPath()
+	if err != nil {
+		return "", err
+	}
 
 	tokens := strings.Split(pciPath.path, "/")
 

device_amd64.go

@@ -8,12 +8,14 @@
 package main
 
 const (
-	rootBusPath = "/devices/pci0000:00"
-
 	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 	// The Linux kernel's core ACPI subsystem creates struct acpi_device
 	// objects for ACPI namespace objects representing devices, power resources
 	// processors, thermal zones. Those objects are exported to user space via
 	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 	acpiDevPath = "/devices/LNXSYSTM"
 )
+
+func createRootBusPath() (string, error) {
+	return "/devices/pci0000:00", nil
+}

device_arm64.go

@@ -6,13 +6,37 @@
 
 package main
 
-const (
-	rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+)
 
+const (
 	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 	// The Linux kernel's core ACPI subsystem creates struct acpi_device
 	// objects for ACPI namespace objects representing devices, power resources
 	// processors, thermal zones. Those objects are exported to user space via
 	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 	acpiDevPath = "/devices/LNXSYSTM"
 )
+
+func createRootBusPath() (string, error) {
+	startRootBusPath := "/devices/platform"
+	endRootBusPath := "/pci0000:00"
+	sysStartRootBusPath := filepath.Join(sysfsDir, startRootBusPath)
+	files, err := ioutil.ReadDir(sysStartRootBusPath)
+	if err != nil {
+		return "", fmt.Errorf("Error reading %s: %s", sysStartRootBusPath, err)
+	}
+
+	// find out the directory end with ".pcie"
+	for _, file := range files {
+		if strings.HasSuffix(file.Name(), ".pcie") && file.IsDir() {
+			return filepath.Join(startRootBusPath, file.Name(), endRootBusPath), nil
+		}
+	}
+
+	return "", fmt.Errorf("no pcie bus found under %s", sysStartRootBusPath)
+}

device_ppc64le.go

@@ -8,12 +8,14 @@
 package main
 
 const (
-	rootBusPath = "/devices/pci0000:00"
-
 	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 	// The Linux kernel's core ACPI subsystem creates struct acpi_device
 	// objects for ACPI namespace objects representing devices, power resources
 	// processors, thermal zones. Those objects are exported to user space via
 	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 	acpiDevPath = "/devices/LNXSYSTM"
 )
+
+func createRootBusPath() (string, error) {
+	return "/devices/pci0000:00", nil
+}

device_s390x.go

@@ -8,12 +8,14 @@
 package main
 
 const (
-	rootBusPath = "/devices/css0"
-
 	// From https://www.kernel.org/doc/Documentation/acpi/namespace.txt
 	// The Linux kernel's core ACPI subsystem creates struct acpi_device
 	// objects for ACPI namespace objects representing devices, power resources
 	// processors, thermal zones. Those objects are exported to user space via
 	// sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00
 	acpiDevPath = "/devices/LNXSYSTM"
 )
+
+func createRootBusPath() (string, error) {
+	return "/devices/css0", nil
+}

device_test.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -93,6 +94,17 @@ func TestVirtioBlkDeviceHandlerEmptyLinuxDevicesSpecFailure(t *testing.T) {
 }
 
 func TestPciPathToSysfs(t *testing.T) {
+	var rootBusPath string
+	var err error
+
+	if runtime.GOARCH == "arm64" {
+		rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+	} else {
+		rootBusPath, err = createRootBusPath()
+		if err != nil {
+			t.Fatal(t, err)
+		}
+	}
 	testDir, err := ioutil.TempDir("", "kata-agent-tmp-")
 	if err != nil {
 		t.Fatal(t, err)

mount_test.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"syscall"
 	"testing"
 
@@ -210,6 +211,17 @@ func TestVirtioBlkStorageDeviceFailure(t *testing.T) {
 
 func TestVirtioBlkStorageHandlerSuccessful(t *testing.T) {
 	skipUnlessRoot(t)
+	var rootBusPath string
+	var err error
+
+	if runtime.GOARCH == "arm64" {
+		rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00"
+	} else {
+		rootBusPath, err = createRootBusPath()
+		if err != nil {
+			t.Fatal(t, err)
+		}
+	}
 
 	testDir, err := ioutil.TempDir("", "kata-agent-tmp-")
 	if err != nil {