Update PIN_PAGES tests

- flags = 0 is now valid.
- Test contiguous mappings backed by huge pages.
- Test discontiguous mappings.

test/enumeration.cpp

@@ -112,6 +112,19 @@ std::map<dev_t, PciBusDeviceFunction> EnumeratePciDevices()
     return devices;
 }
 
+bool IsIommuTranslated(PciBusDeviceFunction bdf)
+{
+    try
+    {
+        auto iommu_type = read_file(sysfs_dir_for_bdf(bdf) + "/iommu_group/type");
+        return iommu_type.substr(0, 3) == "DMA";
+    }
+    catch (...)
+    {
+        return false;
+    }
+}
+
 std::vector<EnumeratedDevice> EnumerateDevices(void)
 {
     auto driver_devices = EnumerateDriverDevices();
@@ -133,7 +146,7 @@ std::vector<EnumeratedDevice> EnumerateDevices(void)
     for (const auto &driver_dev : driver_devices)
     {
         dev_t dev = driver_dev.first;
-        devices.push_back({driver_dev.second, pci_devices[dev], dev});
+        devices.push_back({driver_dev.second, pci_devices[dev], dev, IsIommuTranslated(pci_devices[dev])});
     }
 
     return devices;

test/enumeration.h

@@ -15,6 +15,7 @@ struct EnumeratedDevice
     std::string path;
     PciBusDeviceFunction location;
     dev_t node;
+    bool iommu_translated;
 };
 
 std::vector<EnumeratedDevice> EnumerateDevices(void);

test/pin_pages.cpp

@@ -1,21 +1,22 @@
 // SPDX-FileCopyrightText: © 2023 Tenstorrent Inc.
 // SPDX-License-Identifier: GPL-2.0-only
 
-// Verify that pin pages requires TENSTORRENT_PIN_PAGES_CONTIGUOUS.
+// Verify that pin pages accepts flags = 0 or TENSTORRENT_PIN_PAGES_CONTIGUOUS.
 // Verify that pin pages rejects any other flags.
 // Verify that pin pages rejects size == 0 and size not multiple of page size.
 // Verify that pin pages rejects an unmapped range, a partially unmapped range.
 // Verify that pin pages accepts a single page.
 // Verify that pin pages can simultaneously pin many ranges.
+// Verify that pin pages can pin multiple pages if they are contiguous.
+// Verify that pin pages can pin discontiguous memory if and only if IOMMU is enabled.
 
 #include <iostream>
-#include <fstream>
 #include <memory>
-#include <optional>
 #include <regex>
+#include <stdexcept>
 #include <string>
 #include <vector>
-#include <cerrno>
+#include <cmath>
 #include <cstddef>
 #include <cstdlib>
 
@@ -59,20 +60,21 @@ void VerifyPinPagesSimple(const EnumeratedDevice &dev)
     void *p = std::aligned_alloc(page_size, page_size);
     std::unique_ptr<void, Freer> page(p);
 
+    static const unsigned int flags[] = { 0, TENSTORRENT_PIN_PAGES_CONTIGUOUS };
+    for (auto f : flags)
     {
         DevFd dev_fd(dev.path);
 
         zero(&pin_pages);
         pin_pages.in.output_size_bytes = sizeof(pin_pages.out);
 
-        pin_pages.in.flags = TENSTORRENT_PIN_PAGES_CONTIGUOUS;
+        pin_pages.in.flags = f;
         pin_pages.in.virtual_address = reinterpret_cast<uintptr_t>(page.get());
         pin_pages.in.size = page_size;
 
         if (ioctl(dev_fd.get(), TENSTORRENT_IOCTL_PIN_PAGES, &pin_pages) != 0)
             THROW_TEST_FAILURE("PIN_PAGES failed single-page pin.");
     }
-
 }
 
 void VerifyPinPagesBadFlags(const EnumeratedDevice &dev)
@@ -84,20 +86,6 @@ void VerifyPinPagesBadFlags(const EnumeratedDevice &dev)
     void *p = std::aligned_alloc(page_size, page_size);
     std::unique_ptr<void, Freer> page(p);
 
-    {
-        DevFd dev_fd(dev.path);
-
-        zero(&pin_pages);
-        pin_pages.in.output_size_bytes = sizeof(pin_pages.out);
-
-        pin_pages.in.flags = 0;
-        pin_pages.in.virtual_address = reinterpret_cast<uintptr_t>(page.get());
-        pin_pages.in.size = page_size;
-
-        if (ioctl(dev_fd.get(), TENSTORRENT_IOCTL_PIN_PAGES, &pin_pages) != -1)
-            THROW_TEST_FAILURE("PIN_PAGES succeeded with flags = 0.");
-    }
-
     {
         DevFd dev_fd(dev.path);
 
@@ -222,6 +210,151 @@ void VerifyPinPagesMultipleRanges(const EnumeratedDevice &dev)
     }
 }
 
+void VerifyPinPagesContiguous(const EnumeratedDevice &dev)
+{
+    // To verify contiguous mappings, we need to use a hugepage.
+
+    static const std::string hugepage_parent_dir = "/sys/kernel/mm/hugepages";
+    static const std::regex hugepage_subdir_re("hugepages-([0-9]+)kB");
+
+    auto hugepage_subdirs = list_dir(hugepage_parent_dir);
+
+    bool successful_allocation = false;
+
+    for (const auto &hugepage_subdir : hugepage_subdirs)
+    {
+        std::smatch m;
+        if (std::regex_match(hugepage_subdir, m, hugepage_subdir_re))
+        {
+            auto hugepage_size_kb = std::stoul(m[1]);
+            std::size_t hugepage_size = static_cast<std::size_t>(hugepage_size_kb) * 1024;
+            unsigned int huge_size_log2 = std::log2(hugepage_size);
+
+            void *m = mmap(nullptr, hugepage_size, PROT_READ | PROT_WRITE,
+                           MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | (huge_size_log2 << MAP_HUGE_SHIFT),
+                           -1, 0);
+
+            if (m != MAP_FAILED)
+            {
+                successful_allocation = true;
+
+                std::unique_ptr<void, Unmapper> mapping(m, Unmapper{hugepage_size / getpagesize()});
+
+                DevFd dev_fd(dev.path);
+
+                struct tenstorrent_pin_pages pin_pages;
+                zero(&pin_pages);
+                pin_pages.in.output_size_bytes = sizeof(pin_pages.out);
+
+                pin_pages.in.flags = TENSTORRENT_PIN_PAGES_CONTIGUOUS;
+                pin_pages.in.virtual_address = reinterpret_cast<uintptr_t>(mapping.get());
+                pin_pages.in.size = hugepage_size;
+
+                if (ioctl(dev_fd.get(), TENSTORRENT_IOCTL_PIN_PAGES, &pin_pages) != 0)
+                    THROW_TEST_FAILURE("Hugepage pin failed.");
+            }
+        }
+    }
+
+    if (!successful_allocation)
+    {
+        std::cout << "No huge pages could be allocated for VerifyPinPagesContiguous, test skipped.\n";
+    }
+}
+
+void VerifyPinPagesNotContiguous(const EnumeratedDevice &dev)
+{
+    // How do we get 2 pages that are not physically contiguous?
+    // Create a temporary file large enough for 2 pages, mmap it MAP_SHARED and touch the pages.
+    // Create a second mapping of the file with the order of the pages swapped.
+    // It's not possible for the pages to be physically contiguous in both mappings.
+
+    auto page_size = getpagesize();
+
+    // Create the 2-page temporary file.
+    int temp_fd = make_anonymous_temp();
+
+    if (ftruncate(temp_fd, 2 * page_size))
+        throw_system_error("failed to resize temporary file.");
+
+    // First mapping
+    void *m = mmap(nullptr, 2 * page_size, PROT_READ | PROT_WRITE, MAP_SHARED, temp_fd, 0);
+    if (m == MAP_FAILED)
+        throw_system_error("2-page temporary file mapping failed.");
+
+    std::unique_ptr<unsigned char, Unmapper> first_mapping(static_cast<unsigned char*>(m), Unmapper{2});
+
+    first_mapping.get()[0] = 1;
+    first_mapping.get()[page_size] = 2;
+
+    // Second mapping
+    // reserve 2 pages of VA
+    m = mmap(nullptr, 2 * page_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+    std::unique_ptr<unsigned char, Unmapper> second_mapping(static_cast<unsigned char*>(m), Unmapper{2});
+
+    // map the pages from the file into that space, but in reverse order
+    if (mmap(second_mapping.get(), page_size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, temp_fd, page_size) == MAP_FAILED)
+        throw_system_error("remapping temporary file (first page) failed.");
+
+    if (mmap(second_mapping.get() + page_size, page_size, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, temp_fd, 0) == MAP_FAILED)
+        throw_system_error("remapping temporary file (second page) failed.");
+
+    if (second_mapping.get()[0] != 2 || second_mapping.get()[page_size] != 1)
+        throw std::logic_error("Reverse mapping was not set up correctly in VerifyPinPagesNotContiguous.");
+
+    unsigned int flags = dev.iommu_translated ? 0 : TENSTORRENT_PIN_PAGES_CONTIGUOUS;
+
+    bool first_pin_succeeded = false;
+    bool second_pin_succeeded = false;
+
+    struct tenstorrent_pin_pages pin_pages;
+
+    {
+        DevFd dev_fd(dev.path);
+
+        zero(&pin_pages);
+        pin_pages.in.output_size_bytes = sizeof(pin_pages.out);
+
+        pin_pages.in.flags = flags;
+        pin_pages.in.virtual_address = reinterpret_cast<uintptr_t>(first_mapping.get());
+        pin_pages.in.size = 2 * page_size;
+
+        first_pin_succeeded = (ioctl(dev_fd.get(), TENSTORRENT_IOCTL_PIN_PAGES, &pin_pages) != -1);
+    }
+
+    {
+        DevFd dev_fd(dev.path);
+
+        zero(&pin_pages);
+        pin_pages.in.output_size_bytes = sizeof(pin_pages.out);
+
+        pin_pages.in.flags = flags;
+        pin_pages.in.virtual_address = reinterpret_cast<uintptr_t>(second_mapping.get());
+        pin_pages.in.size = 2 * page_size;
+
+        second_pin_succeeded = (ioctl(dev_fd.get(), TENSTORRENT_IOCTL_PIN_PAGES, &pin_pages) != -1);
+    }
+
+    if (dev.iommu_translated)
+    {
+        // With IOMMU enabled, both pin cases must pass, discontiguous pinnings are allowed.
+        if (!first_pin_succeeded && !second_pin_succeeded)
+            THROW_TEST_FAILURE("Both PIN_PAGES failed in VerifyPinPagesNotContiguous.");
+
+        if (!first_pin_succeeded)
+            THROW_TEST_FAILURE("First PIN_PAGES (presumably contiguous) failed in VerifyPinPagesNotContiguous.");
+
+        if (!second_pin_succeeded)
+            THROW_TEST_FAILURE("Second PIN_PAGES (presumably discontiguous) failed in VerifyPinPagesNotContiguous.");
+    }
+    else
+    {
+        // With IOMMU disabled, at most one can pass. (Both can fail, the pages might not be contiguous.)
+        if (first_pin_succeeded && second_pin_succeeded)
+            THROW_TEST_FAILURE("PIN_PAGES passed on discontiguous pages.");
+    }
+}
+
 }
 
 void TestPinPages(const EnumeratedDevice &dev)
@@ -231,4 +364,6 @@ void TestPinPages(const EnumeratedDevice &dev)
     VerifyPinPagesBadSize(dev);
     VerifyPinPagesNoUnmapped(dev);
     VerifyPinPagesMultipleRanges(dev);
+    VerifyPinPagesContiguous(dev);
+    VerifyPinPagesNotContiguous(dev);
 }

test/util.cpp

@@ -9,6 +9,7 @@
 #include <cstddef>
 #include <cstring>
 
+#include <stdlib.h>
 #include <sys/types.h>
 #include <dirent.h>
 #include <unistd.h>
@@ -146,3 +147,27 @@ std::string PciBusDeviceFunction::format() const
 
     return buf;
 }
+
+int make_anonymous_temp()
+{
+    const char *tmpdir_env = getenv("TMPDIR");
+
+    std::string template_str = (tmpdir_env && tmpdir_env[0]) ? tmpdir_env : "/tmp/";
+    if (template_str.back() != '/')
+    {
+        template_str.push_back('/');
+    }
+
+    template_str += "ttkmd_test_XXXXXX";
+
+    std::vector<char> filename_buf(template_str.begin(), template_str.end());
+    filename_buf.push_back('\0');
+
+    int fd = mkstemp(filename_buf.data());
+    if (fd == -1)
+        throw_system_error("creating temporary file.");
+
+    unlink(filename_buf.data());
+
+    return fd;
+}

test/util.h

@@ -42,3 +42,5 @@ std::string readlink_str(const std::string &link_name);
 std::string sysfs_dir_for_bdf(PciBusDeviceFunction bdf);
 
 unsigned page_size();
+
+int make_anonymous_temp();