From d66fcb8ae3c4de680cf2d403d6f72a1a23be9f73 Mon Sep 17 00:00:00 2001 From: Jianyong Wu Date: Fri, 23 Oct 2020 17:44:30 +0800 Subject: [PATCH] rootBusPath: create rootBusPath dynamically. Currently, rootBusPath is set as a constant value. Bus this pcie bus path is not alway static, eg. the rootBus on arm64 is "/devices/platform/4010000000.pcie/pci0000:00". The address part of "4010000000.pcie" may vary with the maxMem in qemu so it should not be a fixed value. For exmaple: HIGH PCIE address reserved in qemu on virt is from 0 to 512G. The lower part of address may be allocated to normal memory. In general, normal memory is largely less than 256G, so the base address of HIGH PCIE can be 256G. But, in case the maxmem is large enough, like around 256G, the base address of HIGH PCIE must be increase, eg. 300G. In the current implementation of kata-runtime, the maxmem in qemu is the host memory size. So if the host memory size is large enough, the prefix of the pci device path name will be different from rootBusPath set in kata-agent which can lead to failure on device hotplug. This patch offer a mechanism to create rootBusPath dynamically but only give implemention for arm64 and return the default value of rootBusPath for other arch. Fixes: #859 Signed-off-by: Jianyong Wu --- agent.go | 5 +++++ device.go | 4 ++++ device_amd64.go | 6 ++++-- device_arm64.go | 28 ++++++++++++++++++++++++++-- device_ppc64le.go | 6 ++++-- device_s390x.go | 6 ++++-- device_test.go | 12 ++++++++++++ mount_test.go | 12 ++++++++++++ 8 files changed, 71 insertions(+), 8 deletions(-) diff --git a/agent.go b/agent.go index 4c0ff340bf..7c9a758e47 100644 --- a/agent.go +++ b/agent.go @@ -709,6 +709,11 @@ func (s *sandbox) waitForStopServer() { func (s *sandbox) listenToUdevEvents() { fieldLogger := agentLog.WithField("subsystem", "udevlistener") + rootBusPath, err := createRootBusPath() + if err != nil { + fieldLogger.Warnf("Error creating root bus path") + return + } uEvHandler, err := uevent.NewHandler() if err != nil { diff --git a/device.go b/device.go index 994aa39705..478809c134 100644 --- a/device.go +++ b/device.go @@ -109,6 +109,10 @@ func rescanPciBus() error { func pciPathToSysfsImpl(pciPath PciPath) (string, error) { var relPath string bus := "0000:00" + rootBusPath, err := createRootBusPath() + if err != nil { + return "", err + } tokens := strings.Split(pciPath.path, "/") diff --git a/device_amd64.go b/device_amd64.go index fe9f8b785d..02845aa6cb 100644 --- a/device_amd64.go +++ b/device_amd64.go @@ -8,8 +8,6 @@ package main const ( - rootBusPath = "/devices/pci0000:00" - // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources @@ -17,3 +15,7 @@ const ( // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 acpiDevPath = "/devices/LNXSYSTM" ) + +func createRootBusPath() (string, error) { + return "/devices/pci0000:00", nil +} diff --git a/device_arm64.go b/device_arm64.go index 11c4e70d84..abcae0788c 100644 --- a/device_arm64.go +++ b/device_arm64.go @@ -6,9 +6,14 @@ package main -const ( - rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" +) +const ( // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources @@ -16,3 +21,22 @@ const ( // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 acpiDevPath = "/devices/LNXSYSTM" ) + +func createRootBusPath() (string, error) { + startRootBusPath := "/devices/platform" + endRootBusPath := "/pci0000:00" + sysStartRootBusPath := filepath.Join(sysfsDir, startRootBusPath) + files, err := ioutil.ReadDir(sysStartRootBusPath) + if err != nil { + return "", fmt.Errorf("Error reading %s: %s", sysStartRootBusPath, err) + } + + // find out the directory end with ".pcie" + for _, file := range files { + if strings.HasSuffix(file.Name(), ".pcie") && file.IsDir() { + return filepath.Join(startRootBusPath, file.Name(), endRootBusPath), nil + } + } + + return "", fmt.Errorf("no pcie bus found under %s", sysStartRootBusPath) +} diff --git a/device_ppc64le.go b/device_ppc64le.go index fe9f8b785d..02845aa6cb 100644 --- a/device_ppc64le.go +++ b/device_ppc64le.go @@ -8,8 +8,6 @@ package main const ( - rootBusPath = "/devices/pci0000:00" - // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources @@ -17,3 +15,7 @@ const ( // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 acpiDevPath = "/devices/LNXSYSTM" ) + +func createRootBusPath() (string, error) { + return "/devices/pci0000:00", nil +} diff --git a/device_s390x.go b/device_s390x.go index e28074c44d..d9ad561798 100644 --- a/device_s390x.go +++ b/device_s390x.go @@ -8,8 +8,6 @@ package main const ( - rootBusPath = "/devices/css0" - // From https://www.kernel.org/doc/Documentation/acpi/namespace.txt // The Linux kernel's core ACPI subsystem creates struct acpi_device // objects for ACPI namespace objects representing devices, power resources @@ -17,3 +15,7 @@ const ( // sysfs as directories in the subtree under /sys/devices/LNXSYSTM:00 acpiDevPath = "/devices/LNXSYSTM" ) + +func createRootBusPath() (string, error) { + return "/devices/css0", nil +} diff --git a/device_test.go b/device_test.go index 34f5622a04..2259e57560 100644 --- a/device_test.go +++ b/device_test.go @@ -13,6 +13,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "testing" @@ -93,6 +94,17 @@ func TestVirtioBlkDeviceHandlerEmptyLinuxDevicesSpecFailure(t *testing.T) { } func TestPciPathToSysfs(t *testing.T) { + var rootBusPath string + var err error + + if runtime.GOARCH == "arm64" { + rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" + } else { + rootBusPath, err = createRootBusPath() + if err != nil { + t.Fatal(t, err) + } + } testDir, err := ioutil.TempDir("", "kata-agent-tmp-") if err != nil { t.Fatal(t, err) diff --git a/mount_test.go b/mount_test.go index da00a4654e..b0f6825341 100644 --- a/mount_test.go +++ b/mount_test.go @@ -12,6 +12,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "syscall" "testing" @@ -210,6 +211,17 @@ func TestVirtioBlkStorageDeviceFailure(t *testing.T) { func TestVirtioBlkStorageHandlerSuccessful(t *testing.T) { skipUnlessRoot(t) + var rootBusPath string + var err error + + if runtime.GOARCH == "arm64" { + rootBusPath = "/devices/platform/4010000000.pcie/pci0000:00" + } else { + rootBusPath, err = createRootBusPath() + if err != nil { + t.Fatal(t, err) + } + } testDir, err := ioutil.TempDir("", "kata-agent-tmp-") if err != nil {