Skip to content

Commit

Permalink
Add Feature: Add interrupt and interrupt coalescing support
Browse files Browse the repository at this point in the history
  • Loading branch information
byjiang1996 committed Dec 15, 2019
1 parent 09a21a4 commit 864e960
Show file tree
Hide file tree
Showing 17 changed files with 371 additions and 26 deletions.
35 changes: 34 additions & 1 deletion dpdk/drivers/bus/pci/linux/pci.c
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ rte_pci_map_device(struct rte_pci_device *dev)
break;
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
if (rte_eal_using_phys_addrs()) {
/* map resources for devices that use uio */
ret = pci_uio_map_resource(dev);
Expand Down Expand Up @@ -106,6 +107,7 @@ rte_pci_unmap_device(struct rte_pci_device *dev)
break;
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
/* unmap resources for devices that use uio */
pci_uio_unmap_resource(dev);
break;
Expand Down Expand Up @@ -329,6 +331,8 @@ pci_scan_one(const char *dirname, const struct rte_pci_addr *addr)
dev->kdrv = RTE_KDRV_IGB_UIO;
else if (!strcmp(driver, "uio_pci_generic"))
dev->kdrv = RTE_KDRV_UIO_GENERIC;
else if (!strcmp(driver, "byuio_pci_generic"))
dev->kdrv = RTE_KDRV_BYUIO_GENERIC;
else
dev->kdrv = RTE_KDRV_UNKNOWN;
} else
Expand Down Expand Up @@ -549,7 +553,8 @@ pci_one_device_bound_uio(void)
continue;

if (dev->kdrv == RTE_KDRV_IGB_UIO ||
dev->kdrv == RTE_KDRV_UIO_GENERIC) {
dev->kdrv == RTE_KDRV_UIO_GENERIC ||
dev->kdrv == RTE_KDRV_BYUIO_GENERIC) {
return 1;
}
}
Expand Down Expand Up @@ -704,6 +709,28 @@ rte_pci_get_iommu_class(void)
return RTE_IOVA_PA;
}

/* Read PCI interrupt events. */
int rte_pci_read_intr(const struct rte_pci_device *device,
void *buf, size_t len)
{
char devname[RTE_DEV_NAME_MAX_LEN] = "";
const struct rte_intr_handle *intr_handle = &device->intr_handle;

switch (device->kdrv) {
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
len = sizeof(uint32_t);
case RTE_KDRV_BYUIO_GENERIC:
return pci_uio_read_intr(intr_handle, buf, len);
default:
rte_pci_device_name(&device->addr, devname,
RTE_DEV_NAME_MAX_LEN);
RTE_LOG(ERR, EAL,
"Unknown driver type for %s\n", devname);
return -1;
}
}

/* Read PCI config space. */
int rte_pci_read_config(const struct rte_pci_device *device,
void *buf, size_t len, off_t offset)
Expand All @@ -714,6 +741,7 @@ int rte_pci_read_config(const struct rte_pci_device *device,
switch (device->kdrv) {
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
return pci_uio_read_config(intr_handle, buf, len, offset);
#ifdef VFIO_PRESENT
case RTE_KDRV_VFIO:
Expand All @@ -738,6 +766,7 @@ int rte_pci_write_config(const struct rte_pci_device *device,
switch (device->kdrv) {
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
return pci_uio_write_config(intr_handle, buf, len, offset);
#ifdef VFIO_PRESENT
case RTE_KDRV_VFIO:
Expand Down Expand Up @@ -828,6 +857,7 @@ rte_pci_ioport_map(struct rte_pci_device *dev, int bar,
ret = pci_uio_ioport_map(dev, bar, p);
break;
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
#if defined(RTE_ARCH_X86)
ret = pci_ioport_map(dev, bar, p);
#else
Expand Down Expand Up @@ -863,6 +893,7 @@ rte_pci_ioport_read(struct rte_pci_ioport *p,
pci_uio_ioport_read(p, data, len, offset);
break;
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
pci_uio_ioport_read(p, data, len, offset);
break;
case RTE_KDRV_NONE:
Expand All @@ -889,6 +920,7 @@ rte_pci_ioport_write(struct rte_pci_ioport *p,
pci_uio_ioport_write(p, data, len, offset);
break;
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
pci_uio_ioport_write(p, data, len, offset);
break;
case RTE_KDRV_NONE:
Expand Down Expand Up @@ -917,6 +949,7 @@ rte_pci_ioport_unmap(struct rte_pci_ioport *p)
ret = pci_uio_ioport_unmap(p);
break;
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
#if defined(RTE_ARCH_X86)
ret = 0;
#else
Expand Down
2 changes: 2 additions & 0 deletions dpdk/drivers/bus/pci/linux/pci_init.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ void pci_uio_free_resource(struct rte_pci_device *dev,
int pci_uio_map_resource_by_index(struct rte_pci_device *dev, int res_idx,
struct mapped_pci_resource *uio_res, int map_idx);

int pci_uio_read_intr(const struct rte_intr_handle *intr_handle,
void *buf, size_t len);
int pci_uio_read_config(const struct rte_intr_handle *intr_handle,
void *buf, size_t len, off_t offs);
int pci_uio_write_config(const struct rte_intr_handle *intr_handle,
Expand Down
9 changes: 9 additions & 0 deletions dpdk/drivers/bus/pci/linux/pci_uio.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ void *pci_map_addr = NULL;

#define OFF_MAX ((uint64_t)(off_t)-1)

int
pci_uio_read_intr(const struct rte_intr_handle *intr_handle,
void *buf, size_t len)
{
if (read(intr_handle->fd, buf, len) < 0)
return -1;
return 0;
}

int
pci_uio_read_config(const struct rte_intr_handle *intr_handle,
void *buf, size_t len, off_t offset)
Expand Down
1 change: 1 addition & 0 deletions dpdk/drivers/bus/pci/pci_common.c
Original file line number Diff line number Diff line change
Expand Up @@ -469,6 +469,7 @@ pci_hot_unplug_handler(struct rte_device *dev)
#endif
case RTE_KDRV_IGB_UIO:
case RTE_KDRV_UIO_GENERIC:
case RTE_KDRV_BYUIO_GENERIC:
case RTE_KDRV_NIC_UIO:
/* BARs resource is invalid, remap it to be safe. */
ret = pci_uio_remap_resource(pdev);
Expand Down
16 changes: 16 additions & 0 deletions dpdk/drivers/bus/pci/rte_bus_pci.h
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,22 @@ RTE_PMD_EXPORT_NAME(nm, __COUNTER__)
*/
void rte_pci_unregister(struct rte_pci_driver *driver);

/**
* Read PCI interrupt events (block waiting interrupts).
*
* @param device
* A pointer to a rte_pci_device structure describing the device
* to use
* @param buf
* A data buffer where the bytes should be read into
* @param len
* The length of the data buffer.
* @return
* Number of bytes read on success, negative on error.
*/
int rte_pci_read_intr(const struct rte_pci_device *device,
void *buf, size_t len);

/**
* Read PCI config space.
*
Expand Down
1 change: 1 addition & 0 deletions dpdk/lib/librte_eal/common/include/rte_dev.h
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ enum rte_kernel_driver {
RTE_KDRV_UIO_GENERIC,
RTE_KDRV_NIC_UIO,
RTE_KDRV_NONE,
RTE_KDRV_BYUIO_GENERIC,
};

/**
Expand Down
12 changes: 12 additions & 0 deletions include/spdk/env.h
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,7 @@ struct spdk_pci_device {
void **mapped_addr, uint64_t *phys_addr, uint64_t *size);
int (*unmap_bar)(struct spdk_pci_device *dev, uint32_t bar,
void *addr);
int (*intr_read)(struct spdk_pci_device *dev, void *value, uint32_t len);
int (*cfg_read)(struct spdk_pci_device *dev, void *value,
uint32_t len, uint32_t offset);
int (*cfg_write)(struct spdk_pci_device *dev, void *value,
Expand Down Expand Up @@ -946,6 +947,17 @@ void spdk_pci_device_detach(struct spdk_pci_device *device);
int spdk_pci_device_attach(struct spdk_pci_driver *driver, spdk_pci_enum_cb enum_cb,
void *enum_ctx, struct spdk_pci_addr *pci_address);

/**
* Batch read PCI interrupt events until cq_tail reaches cq_ptr.
*
* \param dev PCI device.
* \param cq_ptr Completion queue pointer waited.
* \param cqid Completion queue id.
*
* \return 0 on success, -1 on failure.
*/
int spdk_pci_device_intr_read(struct spdk_pci_device *dev, uint16_t cq_ptr, uint16_t cqid);

/**
* Read \c len bytes from the PCI configuration space.
*
Expand Down
41 changes: 41 additions & 0 deletions include/spdk/nvme.h
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,13 @@ struct spdk_nvme_ctrlr_opts {
* Defaults to 'false' (errors are logged).
*/
bool disable_error_logging;

/**
* Enable interrupt in NVMe Completion Queue.
*
* Defaults to 'false' (interrupt disabled).
*/
bool interrupt_enabled;
};

/**
Expand Down Expand Up @@ -1176,6 +1183,31 @@ int spdk_nvme_ctrlr_cmd_io_raw_with_md(struct spdk_nvme_ctrlr *ctrlr,
int32_t spdk_nvme_qpair_process_completions(struct spdk_nvme_qpair *qpair,
uint32_t max_completions);

/**
* Process any outstanding completions by interrupt for I/O submitted on a queue pair.
*
* This call is blocking, i.e. it will wait until totally min_completions processes
* are ready at the time of this function call. It dowait for outstanding commands to finish.
*
* For each completed command, the request's callback function will be called if
* specified as non-NULL when the request was submitted.
*
* The caller must ensure that each queue pair is only used from one thread at a
* time.
*
* This function may be called at any point while the controller is attached to
* the SPDK NVMe driver.
*
* \sa spdk_nvme_cmd_cb
*
* \param qpair Queue pair to check for completions.
* \param min_completions Limit the number of completions to trigger one interrupt.
*
* \return number of completions processed (may be 0) or negated on error.
*/
int32_t spdk_nvme_qpair_interrupt_completions(struct spdk_nvme_qpair *qpair,
uint32_t min_completions);

/**
* Send the given admin command to the NVMe controller.
*
Expand Down Expand Up @@ -1567,6 +1599,15 @@ int spdk_nvme_ctrlr_update_firmware(struct spdk_nvme_ctrlr *ctrlr, void *payload
int slot, enum spdk_nvme_fw_commit_action commit_action,
struct spdk_nvme_status *completion_status);

/**
* Set interrupt enabled in Completion Queue
*
* \param ctrlr NVMe controller to perform firmware operation on.
*
* \return 0 if successfully set, -1 if the default interrupt vector allocated is wrong.
*/
int nvme_ctrlr_set_intr(struct spdk_nvme_ctrlr *ctrlr);

/**
* Return virtual address of PCIe NVM I/O registers
*
Expand Down
15 changes: 15 additions & 0 deletions lib/env_dpdk/pci.c
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ spdk_unmap_bar_rte(struct spdk_pci_device *device, uint32_t bar, void *addr)
return 0;
}

static int
spdk_intr_read_rte(struct spdk_pci_device *dev, void *value, uint32_t len)
{
return rte_pci_read_intr(dev->dev_handle, value, len);
}

static int
spdk_cfg_read_rte(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset)
{
Expand Down Expand Up @@ -337,6 +343,7 @@ spdk_pci_device_init(struct rte_pci_driver *_drv,

dev->map_bar = spdk_map_bar_rte;
dev->unmap_bar = spdk_unmap_bar_rte;
dev->intr_read = spdk_intr_read_rte;
dev->cfg_read = spdk_cfg_read_rte;
dev->cfg_write = spdk_cfg_write_rte;
dev->detach = spdk_detach_rte;
Expand Down Expand Up @@ -603,6 +610,14 @@ spdk_pci_device_get_socket_id(struct spdk_pci_device *dev)
return dev->socket_id;
}

int
spdk_pci_device_intr_read(struct spdk_pci_device *dev, uint16_t cq_ptr, uint16_t cqid)
{
uint32_t buf = cq_ptr;
buf |= 1 << 24;
return dev->intr_read(dev, &buf, cqid);
}

int
spdk_pci_device_cfg_read(struct spdk_pci_device *dev, void *value, uint32_t len, uint32_t offset)
{
Expand Down
7 changes: 7 additions & 0 deletions lib/nvme/nvme_ctrlr.c
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ nvme_ctrlr_get_cmbsz(struct spdk_nvme_ctrlr *ctrlr, union spdk_nvme_cmbsz_regist
&cmbsz->raw);
}

int
nvme_ctrlr_set_intr(struct spdk_nvme_ctrlr *ctrlr)
{
ctrlr->opts.interrupt_enabled = true;
return 0;
}

volatile struct spdk_nvme_registers *
spdk_nvme_ctrlr_get_registers(struct spdk_nvme_ctrlr *ctrlr)
{
Expand Down
1 change: 1 addition & 0 deletions lib/nvme/nvme_internal.h
Original file line number Diff line number Diff line change
Expand Up @@ -1085,6 +1085,7 @@ struct spdk_nvme_ctrlr *spdk_nvme_get_ctrlr_by_trid_unsafe(
int nvme_ ## name ## _qpair_reset(struct spdk_nvme_qpair *qpair); \
int nvme_ ## name ## _qpair_submit_request(struct spdk_nvme_qpair *qpair, struct nvme_request *req); \
int32_t nvme_ ## name ## _qpair_process_completions(struct spdk_nvme_qpair *qpair, uint32_t max_completions); \
int32_t nvme_ ## name ## _qpair_interrupt_completions(struct spdk_nvme_qpair *qpair, uint32_t min_completions); \
void nvme_ ## name ## _admin_qpair_abort_aers(struct spdk_nvme_qpair *qpair); \

DECLARE_TRANSPORT(transport) /* generic transport dispatch functions */
Expand Down
Loading

0 comments on commit 864e960

Please sign in to comment.