Commit f8259c91 authored by Ankit Kumar's avatar Ankit Kumar Committed by Konrad Sztyber
Browse files

env_dpdk: new interfaces for pci device multi interrupt 



To handle multiple interrupts 3 new interfaces have been added

spdk_pci_device_enable_interrupts()
It first checks if the device has multiple interrupt vector capability,
and based on that it creates a bunch of eventfds. Finally interrupt is
enabled by mapping those eventfds to interrupt vector. The interrupt
vector offset 0 is used for device fd, Interrupt vector offset 1 onwards
is mapped to eventfd at index 0 onwards.

spdk_pci_device_disable_interrupts() disables the interrupts and closes
all the eventfds.

spdk_pci_device_get_interrupt_efd_by_index() returns the device fd for
index 0, and returns the eventfd for any index > 0.

Change-Id: Ifef38ce89ce7c0e58652a30a61e0df448858398d
Signed-off-by: default avatarAnkit Kumar <ankit.kumar@samsung.com>
Signed-off-by: default avatarLiu, Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24903


Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Community-CI: Mellanox Build Bot
Community-CI: Community CI Samsung <spdk.community.ci.samsung@gmail.com>
parent 00715c7c

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -16,6 +16,11 @@ to `multipath` field in spdk_bdev_nvme_ctrlr_opts structure passed as a paramete 
If multipathing shall be enabled for nvme bdev, `bdev_opts.multipath` shall be set to `true`. When

`bdev_opts.multipath` is set to `false`, failover mode is enabled.



### env



Added 3 APIs to handle multiple interrupts for PCI device `spdk_pci_device_enable_interrupts()`,

`spdk_pci_device_disable_interrupts()`, and `spdk_pci_device_get_interrupt_efd_by_index()`.



### nvmf



Added public API `spdk_nvmf_send_discovery_log_notice` to send discovery log page

include/spdk/env.h

+34 −0
Original line number Diff line number Diff line
@@ -943,6 +943,40 @@ int spdk_pci_device_disable_interrupt(struct spdk_pci_device *dev); 
 */

int spdk_pci_device_get_interrupt_efd(struct spdk_pci_device *dev);



/**

 * Enable PCI device interrupts, only if VFIO MSI-X is supported.

 * This creates a bunch of event file descriptors, for which VFIO IRQ are set,

 * which then can be used to enable interrupts.

 *

 * \param dev PCI device.

 * \param efd_count Number of event fds to create.

 *

 * \return 0 on success, negative value on error.

 */

int spdk_pci_device_enable_interrupts(struct spdk_pci_device *dev, uint32_t efd_count);



/**

 * Disable PCI device interrupts.

 * This disables the MSI-X interrupts for all the created event file descriptors

 * and frees them.

 *

 * \param dev PCI device.

 *

 * \return 0 on success, negative value on error.

 */

int spdk_pci_device_disable_interrupts(struct spdk_pci_device *dev);



/**

 * Get an event file descriptor associated with a PCI device, for a particular

 * MSI-X index.

 *

 * \param dev PCI device.

 * \param index event file descriptor index.

 *

 * \return Event file descriptor on success, negative value on error.

 */

int spdk_pci_device_get_interrupt_efd_by_index(struct spdk_pci_device *dev, uint32_t index);



/**

 * Get the domain of a PCI device.

 *

lib/env_dpdk/pci.c

+68 −0
Original line number Diff line number Diff line
@@ -808,6 +808,74 @@ spdk_pci_device_get_interrupt_efd(struct spdk_pci_device *dev) 
	return dpdk_pci_device_get_interrupt_efd(dev->dev_handle);

}



int

spdk_pci_device_enable_interrupts(struct spdk_pci_device *dev, uint32_t efd_count)

{

	struct rte_pci_device *rte_dev = dev->dev_handle;

	int rc;



	if (efd_count == 0) {

		SPDK_ERRLOG("Invalid efd_count (%u)\n", efd_count);

		return -EINVAL;

	}



	/* Detect if device has MSI-X capability */

	if (dpdk_pci_device_interrupt_cap_multi(rte_dev) != 1) {

		SPDK_ERRLOG("VFIO MSI-X capability not present for device %s\n",

			    dpdk_pci_device_get_name(rte_dev));

		return -ENOTSUP;

	}



	/* Create event file descriptors */

	rc = dpdk_pci_device_create_interrupt_efds(rte_dev, efd_count);

	if (rc) {

		SPDK_ERRLOG("Can't setup eventfd (%u)\n", efd_count);

		return rc;

	}



	/* Bind each event fd to each interrupt vector */

	rc = dpdk_pci_device_enable_interrupt(rte_dev);

	if (rc) {

		SPDK_ERRLOG("Failed to enable interrupt for PCI device %s\n",

			    dpdk_pci_device_get_name(rte_dev));

		dpdk_pci_device_delete_interrupt_efds(rte_dev);

		return rc;

	}



	return 0;

}



int

spdk_pci_device_disable_interrupts(struct spdk_pci_device *dev)

{

	struct rte_pci_device *rte_dev = dev->dev_handle;

	int rc;



	rc = dpdk_pci_device_disable_interrupt(rte_dev);

	if (rc) {

		SPDK_ERRLOG("Failed to disable interrupt for PCI device %s\n",

			    dpdk_pci_device_get_name(rte_dev));

		return rc;

	}



	dpdk_pci_device_delete_interrupt_efds(rte_dev);



	return 0;

}



int

spdk_pci_device_get_interrupt_efd_by_index(struct spdk_pci_device *dev, uint32_t index)

{

	if (index == 0) {

		return dpdk_pci_device_get_interrupt_efd(dev->dev_handle);

	} else {

		/* Note: The interrupt vector offset starts from 1, and in DPDK these

		 * are mapped to efd index 0 onwards.

		 */

		return dpdk_pci_device_get_interrupt_efd_by_index(dev->dev_handle, index - 1);

	}

}



uint32_t

spdk_pci_device_get_domain(struct spdk_pci_device *dev)

{

lib/env_dpdk/spdk_env_dpdk.map

+3 −0
Original line number Diff line number Diff line
@@ -73,6 +73,9 @@ 
	spdk_pci_device_enable_interrupt;

	spdk_pci_device_disable_interrupt;

	spdk_pci_device_get_interrupt_efd;

	spdk_pci_device_enable_interrupts;

	spdk_pci_device_disable_interrupts;

	spdk_pci_device_get_interrupt_efd_by_index;

	spdk_pci_device_get_domain;

	spdk_pci_device_get_bus;

	spdk_pci_device_get_dev;