vfio functionality

src/apps/basic/basic_apps.lua

@@ -18,7 +18,7 @@ function Source:pull ()
       for i = 1, 1000 do
          local p = packet.allocate()
          packet.add_iovec(p, buffer.allocate(), 60)
-	 app.transmit(o, p)
+         app.transmit(o, p)
       end
    end
 end
@@ -34,7 +34,7 @@ end
 function Join:push () 
    for _, inport in ipairs(self.inputi) do
       for _ = 1,math.min(app.nreadable(inport), app.nwritable(self.output.out)) do
-	 app.transmit(self.output.out, app.receive(inport))
+         app.transmit(self.output.out, app.receive(inport))
       end
    end
 end
@@ -70,9 +70,9 @@ end
 function Sink:push ()
    for _, i in ipairs(self.inputi) do
       for _ = 1, app.nreadable(i) do
-	 local p = app.receive(i)
-	 assert(p.refcount == 1)
-	 packet.deref(p)
+        local p = app.receive(i)
+        assert(p.refcount == 1)
+        packet.deref(p)
       end
    end
 end

src/apps/intel/intel.lua

@@ -16,7 +16,7 @@ module(moduleinstance,package.seeall)
 local ffi = require("ffi")
 local C = ffi.C
 local bit = require("bit")
-local pci = require("pci")
+local bus = require("bus")
 local lib = require("lib")
 local test = require("test")
 local bits, bitset = lib.bits, lib.bitset
@@ -27,7 +27,7 @@ require("intel_h")
 -- FFI definitions for receive and transmit descriptors
 
 -- PCI device ID
-local device = pci.device_info(pciaddress).device
+local device = bus.device_info(pciaddress).device
 
 -- Return a table for protected (bounds-checked) memory access.
 -- 
@@ -101,7 +101,7 @@ local MANC   = 0x05820 / 4 -- Management Control Register (RW / 82571)
 local SWSM   = 0x05B50  / 4 -- Software Semaphore (RW / 82571)
 local EEMNGCTL = 0x01010 / 4 -- MNG EEPROM Control (RW / 82571)
 
-local regs = ffi.cast("uint32_t *", pci.map_pci_memory(pciaddress, 0))
+local regs = ffi.cast("uint32_t *", bus.map_pci_memory(pciaddress, 0))
 
 -- Initialization
 
@@ -124,7 +124,7 @@ end
 
 function init_pci ()
    -- PCI bus mastering has to be enabled for DMA to work.
-   pci.set_bus_master(pciaddress, true)
+   bus.set_bus_master(pciaddress, true)
 end
 
 function init_dma_memory ()

src/apps/intel/intel10g.lua

@@ -13,7 +13,7 @@ local C        = ffi.C
 local lib      = require("core.lib")
 local memory   = require("core.memory")
 local packet   = require("core.packet")
-local pci      = require("lib.hardware.pci")
+local bus      = require("lib.hardware.bus")
 local register = require("lib.hardware.register")
                  require("apps.intel.intel_h")
                  require("core.packet_h")
@@ -27,6 +27,7 @@ num_descriptors = 32 * 1024
 
 function new (pciaddress)
    local dev = { pciaddress = pciaddress, -- PCI device address
+                 info = bus.device_info(pciaddress),
                  r = {},           -- Configuration registers
                  s = {},           -- Statistics registers
                  txdesc = 0,     -- Transmit descriptors (pointer)
@@ -46,8 +47,8 @@ function new (pciaddress)
 end
 
 function open (dev)
-   pci.set_bus_master(dev.pciaddress, true)
-   local base = pci.map_pci_memory(dev.pciaddress, 0)
+   dev.info.set_bus_master(dev.pciaddress, true)
+   local base = dev.info.map_pci_memory(dev.pciaddress, 0)
    register.define(config_registers_desc, dev.r, base)
    register.define(statistics_registers_desc, dev.s, base)
    dev.txpackets = ffi.new("struct packet *[?]", num_descriptors)
@@ -70,9 +71,9 @@ end
 
 function init_dma_memory (dev)
    dev.rxdesc, dev.rxdesc_phy =
-      memory.dma_alloc(num_descriptors * ffi.sizeof(rxdesc_t))
+      dev.info.dma_alloc(num_descriptors * ffi.sizeof(rxdesc_t))
    dev.txdesc, dev.txdesc_phy =
-      memory.dma_alloc(num_descriptors * ffi.sizeof(txdesc_t))
+      dev.info.dma_alloc(num_descriptors * ffi.sizeof(txdesc_t))
    -- Add bounds checking
    dev.rxdesc = lib.bounds_checked(rxdesc_t, dev.rxdesc, 0, num_descriptors)
    dev.txdesc = lib.bounds_checked(txdesc_t, dev.txdesc, 0, num_descriptors)

src/apps/intel/intel_app.lua

@@ -45,7 +45,7 @@ end
 
 -- Report on relevant status and statistics.
 function Intel82599:report ()
-   print("report on intel device")
+   print("report on intel device", self.dev.pciaddress)
    --register.dump(self.dev.r)
    register.dump(self.dev.s, true)
 end

src/core/clib.h

@@ -28,3 +28,8 @@ size_t pwrite(int fd, const void *buf, int count, int offset);
 
 // malloc(3) - allocate dynamic memory
 void *malloc(int size);
+
+// readlink(2) - get link's target
+int64_t readlink(const char *path, char *buf, size_t bufsiz);
+char *dirname(char *path);
+char *basename(char *path);

src/core/lib.lua

@@ -3,6 +3,23 @@ module(...,package.seeall)
 local ffi = require("ffi")
 local C = ffi.C
 
+
+function can_open(filename, mode)
+    mode = mode or 'r'
+    local f = io.open(filename, mode)
+    if f == nil then return false end
+    f:close()
+    return true
+end
+
+function can_read(filename)
+    return can_open(filename, 'r')
+end
+
+function can_write(filename)
+    return can_open(filename, 'w')
+end
+
 --- Return `command` in the Unix shell and read `what` from the result.
 function readcmd (command, what)
    local f = io.popen(command)
@@ -27,6 +44,31 @@ function writefile (filename, value)
    return result
 end
 
+function readlink (path)
+    local buf = ffi.new("char[?]", 512)
+    local len = C.readlink(path, buf, 512)
+    if len < 0 then return nil, ffi.errno() end
+    return ffi.string(buf, len)
+end
+
+function dirname(path)
+    if not path then return path end
+
+    local buf = ffi.new("char[?]", #path+1)
+    ffi.copy(buf, path)
+    local ptr = C.dirname(buf)
+    return ffi.string(ptr)
+end
+
+function basename(path)
+    if not path then return path end
+
+    local buf = ffi.new("char[?]", #path+1)
+    ffi.copy(buf, path)
+    local ptr = C.basename(buf)
+    return ffi.string(ptr)
+end
+
 -- Return the name of the first file in `dir`.
 function firstfile (dir)
    return readcmd("ls -1 "..dir.." 2>/dev/null", "*l")
@@ -152,5 +194,9 @@ function selftest ()
    local data = "\x45\x00\x00\x73\x00\x00\x40\x00\x40\x11\xc0\xa8\x00\x01\xc0\xa8\x00\xc7"
    local cs = csum(data, string.len(data))
    assert(cs == 0xb861, "bad checksum: " .. bit.tohex(cs, 4))
+
+--    assert(readlink('/etc/rc2.d/S99rc.local') == '../init.d/rc.local', "bad readlink")
+--    assert(dirname('/etc/rc2.d/S99rc.local') == '/etc/rc2.d', "wrong dirname")
+--    assert(basename('/etc/rc2.d/S99rc.local') == 'S99rc.local', "wrong basename")
 end
 

src/core/memory.lua

@@ -6,6 +6,13 @@ local C = ffi.C
 local lib = require("core.lib")
 require("core.memory_h")
 
+
+-- hook variables
+
+dma_alloc = nil         -- (size) => ram_ptr, io_address
+allocate_RAM = nil      -- (size) => ram_ptr
+ram_to_io_addr = nil    -- (ram_ptr) => io_address
+
 --- ### Serve small allocations from hugepage "chunks"
 
 -- List of all allocated huge pages: {pointer, physical, size, used}
@@ -32,9 +39,10 @@ end
 
 -- Add a new chunk.
 function allocate_next_chunk ()
-   local ptr = allocate_huge_page()
+   local ptr = assert(allocate_RAM(huge_page_size), "Couldn't allocate a chunk of ram")
+   local mem_phy = assert(ram_to_io_addr(ptr, huge_page_size), "Couln't map a chunk of ram to IO address")
    chunks[#chunks + 1] = { pointer = ffi.cast("char*", ptr),
-                           physical = virtual_to_physical(ptr),
+                           physical = mem_phy,
                            size = huge_page_size,
                            used = 0 }
    local addr = tonumber(ffi.cast("uint64_t",ptr))
@@ -47,19 +55,6 @@ end
 -- Configuration option: Set to false to disable HugeTLB.
 use_hugetlb = true
 
-function allocate_huge_page ()
-   if use_hugetlb then
-      local attempts = 3
-      for i = 1,attempts do
-         local page = C.allocate_huge_page(huge_page_size)
-         if page ~= nil then  return page  else  reserve_new_page()  end
-      end
-      error("Failed to allocate a huge page.")
-   else
-      return C.malloc(huge_page_size)
-   end
-end
-
 function reserve_new_page ()
    set_hugepages(get_hugepages() + 1)
 end
@@ -117,5 +112,21 @@ end
 
 --- This module requires a stable physical-virtual mapping so this is
 --- enforced automatically at load-time.
-assert(C.lock_memory() == 0)
 
+function set_use_physical_memory()
+    ram_to_io_addr = virtual_to_physical
+    assert(C.lock_memory() == 0)     -- let's hope it's not needed anymore
+end
+
+function set_default_allocator(use_hugetlb)
+    if use_hugetlb then
+        allocate_RAM = function(size)
+            for i =1, 3 do
+                local page = C.allocate_huge_page(size)
+                if page ~= nil then return page else reserve_new_page() end
+            end
+        end
+    else
+        allocate_RAM = C.malloc
+    end
+end

src/designs/spammer/spammer.lua

@@ -3,14 +3,15 @@ module(...,package.seeall)
 local app       = require("core.app")
 local buffer    = require("core.buffer")
 local timer     = require("core.timer")
-local pci       = require("lib.hardware.pci")
+local bus       = require("lib.hardware.bus")
 local intel_app = require("apps.intel.intel_app")
 local basic_apps = require("apps.basic.basic_apps")
 
 function main ()
+   bus.scan_devices()
    app.apps.source = app.new(basic_apps.Source:new())
    local nics = 0
-   for _,device in ipairs(pci.devices) do
+   for _,device in ipairs(bus.devices) do
       if device.usable and device.driver == 'apps.intel.intel10g' then
          nics = nics + 1
          local name = "nic"..nics

src/lib/hardware/bus.lua

@@ -0,0 +1,87 @@
+
+--- ### Select bus driver between classic PCI and VFIO
+module(...,package.seeall)
+
+local ffi = require('ffi')
+local C = ffi.C
+
+local lib = require('core.lib')
+local pci = require('lib.hardware.pci')
+local vfio = require('lib.hardware.vfio')
+local memory = require("core.memory")
+
+
+
+devices = {}
+map_devices = {}
+
+function scan_devices ()
+    for _,device in ipairs(lib.files_in_directory("/sys/bus/pci/devices")) do
+        local info = device_info(device)
+        if info.driver and not map_devices[device] then
+            table.insert(devices, info)
+            map_devices[device] = info
+        end
+    end
+end
+
+function host_has_vfio()
+    local files = lib.files_in_directory('/sys/kernel/iommu_groups/')
+    return files and #files > 0
+end
+
+
+function device_in_vfio(devicepath)
+    local iommu_group = lib.basename(lib.readlink(devicepath..'/iommu_group'))
+    if not iommu_group then return false end
+    local drivername = lib.basename(lib.readlink(devicepath..'/driver'))
+    return drivername == 'vfio-pci'
+end
+
+
+function device_info(pciaddress)
+    if map_devices[pciaddress] then
+        return map_devices[pciaddress]
+    end
+
+    local pcidevpath = pci.path(pciaddress)
+    if device_in_vfio(pcidevpath) then
+        info  = vfio.device_info(pciaddress)
+        info.bus = 'vfio'
+        info.device_info = vfio.device_info
+        info.map_pci_memory = vfio.map_pci_memory
+        info.set_bus_master = vfio.set_bus_master
+        info.dma_alloc = memory.dma_alloc
+    else
+        info = pci.device_info(pciaddress)
+        info.bus = 'pci'
+        info.device_info = pci.device_info
+        info.map_pci_memory = pci.map_pci_memory
+        info.set_bus_master = pci.set_bus_master
+        info.dma_alloc = memory.dma_alloc
+    end
+    return info
+end
+
+function map_pci_memory(pciaddress, n)
+    return map_devices[pciaddress].map_pci_memory(pciaddress, n)
+end
+
+function set_bus_master(pciaddress, enable)
+    return map_devices[pciaddress].set_bus_master(pciaddress, enable)
+end
+
+function selftest ()
+    print("selftest: bus")
+    scan_devices()
+    for _,info in ipairs(devices) do
+        print (string.format("device %s: %s", info.pciaddress, info.bus))
+    end
+end
+
+if host_has_vfio() then
+    memory.ram_to_io_addr = vfio.set_mapping(memory.huge_page_size)
+else
+    memory.set_use_physical_memory()
+end
+memory.set_default_allocator(true)

src/lib/hardware/pci.lua

@@ -70,7 +70,10 @@ end
 --- Force Linux to release the device with `pciaddress`.
 --- The corresponding network interface (e.g. `eth0`) will disappear.
 function unbind_device_from_linux (pciaddress)
-   lib.writefile(path(pciaddress).."/driver/unbind", pciaddress)
+    local p = path(pciaddress).."/driver/unbind"
+    if lib.can_write(p) then
+        lib.writefile(path(pciaddress).."/driver/unbind", pciaddress)
+    end
 end
 
 --- Return a pointer for MMIO access to `device` resource `n`.
@@ -160,8 +163,3 @@ function open_usable_devices (options)
                     report=true}
    port.selftest(options)
 end
-
-function module_init () scan_devices () end
-
-module_init()
-