Commit df4d03f1 authored by Darek Stojaczyk's avatar Darek Stojaczyk Committed by Jim Harris
Browse files

env: encapsulate spdk_pci_device fields 



In order to populate our PCI device list with devices
located behind the VMD, we'll need to fill out those
device structures from within a special VMD driver. That
driver will base on PCI configuration and BAR accesses,
but definitely not on DPDK. We want to put the VMD driver
outside of the env lib, so we're about to provide it with
a direct access to the device struct. Before we do that,
let's group all the env-internal fields into an extra
struct "internal".

The spdk_pci_device struct does actually depend on DPDK
now as it contains an `rte_pci_device *dev_handle` field,
but we can easily break that dependency. The field is only
used as an arguement to DPDK functions, so we can change
its type to void* and let the implicit type conversion do
the magic. After all, the VMD driver will potentially use
it to store its (non-DPDK) data as well.

Change-Id: I425d6dfa7af13e022f5377ceaff39efbd4a01b3d
Signed-off-by: default avatarDarek Stojaczyk <dariusz.stojaczyk@intel.com>
Reviewed-on: https://review.gerrithub.io/435799


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Chandler-Test-Pool: SPDK Automated Test System <sys_sgsw@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent 1cbd14ca

lib/env_dpdk/env_internal.h

+7 −4
Original line number Diff line number Diff line
@@ -79,13 +79,16 @@ __attribute__((constructor)) static void pci_drv ## _register(void) \ 
}



struct spdk_pci_device {

	struct rte_pci_device		*dev_handle;

	struct spdk_pci_driver		*driver;

	void				*dev_handle;

	struct spdk_pci_addr		addr;

	struct spdk_pci_id		id;

	int				socket_id;



	struct _spdk_pci_device_internal {

		struct spdk_pci_driver		*driver;

		bool				attached;

		TAILQ_ENTRY(spdk_pci_device)	tailq;

	} internal;

};



struct spdk_pci_driver {

lib/env_dpdk/pci.c

+16 −15
Original line number Diff line number Diff line
@@ -108,7 +108,6 @@ spdk_pci_device_init(struct rte_pci_driver *_drv, 
	}



	dev->dev_handle = _dev;

	dev->driver = driver;



	dev->addr.domain = _dev->addr.domain;

	dev->addr.bus = _dev->addr.bus;
@@ -120,16 +119,18 @@ spdk_pci_device_init(struct rte_pci_driver *_drv, 
	dev->id.subdevice_id = _dev->id.subsystem_device_id;

	dev->socket_id = _dev->device.numa_node;



	dev->internal.driver = driver;



	if (driver->cb_fn != NULL) {

		rc = driver->cb_fn(driver->cb_arg, dev);

		if (rc != 0) {

			free(dev);

			return rc;

		}

		dev->attached = true;

		dev->internal.attached = true;

	}



	TAILQ_INSERT_TAIL(&g_pci_devices, dev, tailq);

	TAILQ_INSERT_TAIL(&g_pci_devices, dev, internal.tailq);

	spdk_vtophys_pci_device_added(dev->dev_handle);

	return 0;

}
@@ -139,19 +140,19 @@ spdk_pci_device_fini(struct rte_pci_device *_dev) 
{

	struct spdk_pci_device *dev;



	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {

	TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) {

		if (dev->dev_handle == _dev) {

			break;

		}

	}



	if (dev == NULL || dev->attached) {

	if (dev == NULL || dev->internal.attached) {

		/* The device might be still referenced somewhere in SPDK. */

		return -1;

	}



	spdk_vtophys_pci_device_removed(dev->dev_handle);

	TAILQ_REMOVE(&g_pci_devices, dev, tailq);

	TAILQ_REMOVE(&g_pci_devices, dev, internal.tailq);

	free(dev);

	return 0;
@@ -162,8 +163,8 @@ spdk_pci_device_detach(struct spdk_pci_device *dev) 
{

	struct rte_pci_device *device = dev->dev_handle;



	assert(dev->attached);

	dev->attached = false;

	assert(dev->internal.attached);

	dev->internal.attached = false;

#if RTE_VERSION >= RTE_VERSION_NUM(18, 11, 0, 0)

	char bdf[32];

	int i = 0, rc;
@@ -204,21 +205,21 @@ spdk_pci_device_attach(struct spdk_pci_driver *driver, 


	pthread_mutex_lock(&g_pci_mutex);



	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {

	TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) {

		if (spdk_pci_addr_compare(&dev->addr, pci_address) == 0) {

			break;

		}

	}



	if (dev != NULL && dev->driver == driver) {

		if (dev->attached) {

	if (dev != NULL && dev->internal.driver == driver) {

		if (dev->internal.attached) {

			pthread_mutex_unlock(&g_pci_mutex);

			return -1;

		}



		rc = enum_cb(enum_ctx, dev);

		if (rc == 0) {

			dev->attached = true;

			dev->internal.attached = true;

		}

		pthread_mutex_unlock(&g_pci_mutex);

		return rc;
@@ -278,14 +279,14 @@ spdk_pci_enumerate(struct spdk_pci_driver *driver, 


	pthread_mutex_lock(&g_pci_mutex);



	TAILQ_FOREACH(dev, &g_pci_devices, tailq) {

		if (dev->attached || dev->driver != driver) {

	TAILQ_FOREACH(dev, &g_pci_devices, internal.tailq) {

		if (dev->internal.attached || dev->internal.driver != driver) {

			continue;

		}



		rc = enum_cb(enum_ctx, dev);

		if (rc == 0) {

			dev->attached = true;

			dev->internal.attached = true;

		} else if (rc < 0) {

			pthread_mutex_unlock(&g_pci_mutex);

			return -1;