Commit 047c5aaa authored by Daniel Verkamp's avatar Daniel Verkamp
Browse files

ioat: refactor ioat_attach() API into ioat_probe() 



Similar to the NVMe API change, this allows better abstraction of the
PCI subsystem.

Change-Id: I2b84d9c3c498a08d4451b4ff27d0865f0456c210
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 53d6437b

examples/ioat/perf/perf.c

+35 −53
Original line number Diff line number Diff line
@@ -142,33 +142,10 @@ ioat_done(void *cb_arg) 
	}

}



static int

ioat_init(void)

static bool

probe_cb(void *cb_ctx, void *pdev)

{

	struct pci_device_iterator *iter;

	struct pci_device *pci_dev;

	int err = 0;

	struct pci_id_match match;

	struct ioat_device *dev;



	pci_system_init();

	TAILQ_INIT(&g_devices);



	match.vendor_id		= PCI_MATCH_ANY;

	match.subvendor_id	= PCI_MATCH_ANY;

	match.subdevice_id	= PCI_MATCH_ANY;

	match.device_id		= PCI_MATCH_ANY;

	match.device_class	= 0x088000;

	match.device_class_mask	= 0xFFFFFF;



	iter = pci_id_match_iterator_create(&match);



	while ((pci_dev = pci_device_next(iter)) != NULL) {

		/* Check if the PCI devices is a supported IOAT channel. */

		if (!(ioat_pci_device_match_id(pci_dev->vendor_id,

					       pci_dev->device_id))) {

			continue;

		}

	struct pci_device *pci_dev = pdev;



	printf(" Found matching device at %d:%d:%d "

	       "vendor:0x%04x device:0x%04x\n   name:%s\n",
@@ -178,34 +155,39 @@ ioat_init(void) 


	if (pci_device_has_non_uio_driver(pci_dev)) {

		printf("Device has non-uio kernel driver, skipping...\n");

			continue;

		return false;

	}



	return true;

}



		pci_device_probe(pci_dev);

static void

attach_cb(void *cb_ctx, void *pdev, struct ioat_channel *ioat)

{

	struct ioat_device *dev;



	dev = rte_malloc(NULL, sizeof(*dev), 0);

	if (dev == NULL) {

		printf("Failed to allocate device struct\n");

			err = -1;

			goto cleanup;

		return;

	}



		dev->ioat = ioat_attach(pci_dev);

		if (dev->ioat == NULL) {

			rte_free(dev);

			/* Likely no device found. */

			err = -1;

			goto cleanup;

		}

	dev->ioat = ioat;

	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);

}



cleanup:

	pci_iterator_destroy(iter);

	if (err != 0) {

		ioat_exit();

static int

ioat_init(void)

{

	pci_system_init();

	TAILQ_INIT(&g_devices);



	if (ioat_probe(NULL, probe_cb, attach_cb) != 0) {

		fprintf(stderr, "ioat_probe() failed\n");

		return 1;

	}

	return err;



	return 0;

}



static void

examples/ioat/verify/verify.c

+36 −54
Original line number Diff line number Diff line
@@ -198,33 +198,10 @@ ioat_done(void *cb_arg) 
	}

}



static int

ioat_init(void)

static bool

probe_cb(void *cb_ctx, void *pdev)

{

	struct pci_device_iterator *iter;

	struct pci_device *pci_dev;

	int err = 0;

	struct pci_id_match match;

	struct ioat_device *dev;



	pci_system_init();

	TAILQ_INIT(&g_devices);



	match.vendor_id		= PCI_MATCH_ANY;

	match.subvendor_id	= PCI_MATCH_ANY;

	match.subdevice_id	= PCI_MATCH_ANY;

	match.device_id		= PCI_MATCH_ANY;

	match.device_class	= 0x088000;

	match.device_class_mask	= 0xFFFFFF;



	iter = pci_id_match_iterator_create(&match);



	while ((pci_dev = pci_device_next(iter)) != NULL) {

		/* Check if the PCI devices is a supported IOAT channel. */

		if (!(ioat_pci_device_match_id(pci_dev->vendor_id,

					       pci_dev->device_id))) {

			continue;

		}

	struct pci_device *pci_dev = pdev;



	printf(" Found matching device at %d:%d:%d "

	       "vendor:0x%04x device:0x%04x\n   name:%s\n",
@@ -234,35 +211,40 @@ ioat_init(void) 


	if (pci_device_has_non_uio_driver(pci_dev)) {

		printf("Device has non-uio kernel driver, skipping...\n");

			continue;

		return false;

	}



	return true;

}



		pci_device_probe(pci_dev);

static void

attach_cb(void *cb_ctx, void *pdev, struct ioat_channel *ioat)

{

	struct ioat_device *dev;



	dev = malloc(sizeof(*dev));

	if (dev == NULL) {

		printf("Failed to allocate device struct\n");

			err = -1;

			goto cleanup;

		return;

	}

	memset(dev, 0, sizeof(*dev));



		dev->ioat = ioat_attach(pci_dev);

		if (dev->ioat == NULL) {

			free(dev);

			/* Likely no device found. */

			err = -1;

			goto cleanup;

		}

	dev->ioat = ioat;

	TAILQ_INSERT_TAIL(&g_devices, dev, tailq);

}



cleanup:

	pci_iterator_destroy(iter);

	if (err != 0) {

		ioat_exit();

static int

ioat_init(void)

{

	pci_system_init();

	TAILQ_INIT(&g_devices);



	if (ioat_probe(NULL, probe_cb, attach_cb) != 0) {

		fprintf(stderr, "ioat_probe() failed\n");

		return 1;

	}

	return err;



	return 0;

}



static void

include/spdk/ioat.h

+26 −6
Original line number Diff line number Diff line
@@ -42,26 +42,46 @@ 
#include <stdbool.h>

#include "spdk/pci.h"



/**

 * Opaque handle for a single I/OAT channel returned by \ref ioat_probe().

 */

struct ioat_channel;



/**

 * Signature for callback function invoked when a request is completed.

 */

typedef void (*ioat_callback_t)(void *arg);



/**

 * Returns true if vendor_id and device_id match a known IOAT PCI device ID.

 * Callback for ioat_probe() enumeration.

 *

 * \return true to attach to this device.

 */

bool ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id);

typedef bool (*ioat_probe_cb)(void *cb_ctx, void *pci_dev);



/**

 * Attach an I/OAT PCI device to the I/OAT userspace driver.

 * Callback for ioat_probe() to report a device that has been attached to the userspace I/OAT driver.

 */

typedef void (*ioat_attach_cb)(void *cb_ctx, void *pci_dev, struct ioat_channel *ioat);



/**

 * \brief Enumerate the I/OAT devices attached to the system and attach the userspace I/OAT driver

 * to them if desired.

 *

 * \param probe_cb will be called once per I/OAT device found in the system.

 * \param attach_cb will be called for devices for which probe_cb returned true once the I/OAT

 * controller has been attached to the userspace driver.

 *

 * If called more than once, only devices that are not already attached to the SPDK I/OAT driver

 * will be reported.

 *

 * To stop using the the device and release its associated resources,

 * To stop using the the controller and release its associated resources,

 * call \ref ioat_detach with the ioat_channel instance returned by this function.

 */

struct ioat_channel *ioat_attach(void *device);

int ioat_probe(void *cb_ctx, ioat_probe_cb probe_cb, ioat_attach_cb attach_cb);



/**

 * Detaches specified device returned by \ref ioat_attach() from the I/OAT driver.

 * Detaches specified device returned by \ref ioat_probe() from the I/OAT driver.

 */

int ioat_detach(struct ioat_channel *ioat);

lib/ioat/ioat.c

+68 −69
Original line number Diff line number Diff line
@@ -45,77 +45,14 @@ static __thread struct ioat_channel *ioat_thread_channel; 


struct ioat_driver {

	ioat_mutex_t	lock;

	TAILQ_HEAD(, ioat_channel)	attached_chans;

};



static struct ioat_driver g_ioat_driver = {

	.lock = IOAT_MUTEX_INITIALIZER,

	.attached_chans = TAILQ_HEAD_INITIALIZER(g_ioat_driver.attached_chans),

};



struct pci_device_id {

	uint16_t vendor;

	uint16_t device;

};



static const struct pci_device_id ioat_pci_table[] = {

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB3},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB4},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB5},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB6},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB7},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB3},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB4},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB5},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB6},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_IVB7},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW3},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW4},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW5},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW6},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_HSW7},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX3},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX4},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX5},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX6},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX7},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX8},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDX9},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BWD3},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE0},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE1},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE2},

	{PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_BDXDE3},

};



bool

ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id)

{

	size_t i;

	const struct pci_device_id *ids;



	for (i = 0; i < sizeof(ioat_pci_table) / sizeof(struct pci_device_id); i++) {

		ids = &ioat_pci_table[i];

		if (ids->device == device_id && ids->vendor == vendor_id) {

			return true;

		}

	}

	return false;

}



static uint64_t

ioat_get_chansts(struct ioat_channel *ioat)

{
@@ -526,10 +463,10 @@ ioat_channel_start(struct ioat_channel *ioat) 
	return 0;

}



struct ioat_channel *

/* Caller must hold g_ioat_driver.lock */

static struct ioat_channel *

ioat_attach(void *device)

{

	struct ioat_driver	*driver = &g_ioat_driver;

	struct ioat_channel 	*ioat;

	uint32_t cmd_reg;
@@ -551,13 +488,74 @@ ioat_attach(void *device) 
		return NULL;

	}



	ioat_mutex_lock(&driver->lock);

	SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next);

	ioat_mutex_unlock(&driver->lock);



	return ioat;

}



struct ioat_enum_ctx {

	ioat_probe_cb probe_cb;

	ioat_attach_cb attach_cb;

	void *cb_ctx;

};



/* This function must only be called while holding g_ioat_driver.lock */

static int

ioat_enum_cb(void *ctx, void *pci_dev)

{

	struct ioat_enum_ctx *enum_ctx = ctx;

	struct ioat_channel *ioat;



	/* Verify that this device is not already attached */

	TAILQ_FOREACH(ioat, &g_ioat_driver.attached_chans, tailq) {

		/*

		 * NOTE: This assumes that the PCI abstraction layer will use the same device handle

		 *  across enumerations; we could compare by BDF instead if this is not true.

		 */

		if (pci_dev == ioat->device) {

			return 0;

		}

	}



	if (enum_ctx->probe_cb(enum_ctx->cb_ctx, pci_dev)) {

		/*

		 * Since I/OAT init is relatively quick, just perform the full init during probing.

		 *  If this turns out to be a bottleneck later, this can be changed to work like

		 *  NVMe with a list of devices to initialize in parallel.

		 */

		ioat = ioat_attach(pci_dev);

		if (ioat == NULL) {

			ioat_printf(NULL, "ioat_attach() failed\n");

			return -1;

		}



		TAILQ_INSERT_TAIL(&g_ioat_driver.attached_chans, ioat, tailq);



		enum_ctx->attach_cb(enum_ctx->cb_ctx, pci_dev, ioat);

	}



	return 0;

}



int

ioat_probe(void *cb_ctx, ioat_probe_cb probe_cb, ioat_attach_cb attach_cb)

{

	int rc;

	struct ioat_enum_ctx enum_ctx;



	ioat_mutex_lock(&g_ioat_driver.lock);



	enum_ctx.probe_cb = probe_cb;

	enum_ctx.attach_cb = attach_cb;

	enum_ctx.cb_ctx = cb_ctx;



	rc = ioat_pci_enumerate(ioat_enum_cb, &enum_ctx);



	ioat_mutex_unlock(&g_ioat_driver.lock);



	return rc;

}



int

ioat_detach(struct ioat_channel *ioat)

{
@@ -568,6 +566,7 @@ ioat_detach(struct ioat_channel *ioat) 
	 */

	ioat_mutex_lock(&driver->lock);

	SLIST_REMOVE(&ioat_free_channels, ioat, ioat_channel, next);

	TAILQ_REMOVE(&driver->attached_chans, ioat, tailq);

	ioat_mutex_unlock(&driver->lock);



	ioat_channel_destruct(ioat);

lib/ioat/ioat_impl.h

+96 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include <assert.h>

#include <pthread.h>

#include <stdio.h>

#include <stdbool.h>

#include <rte_malloc.h>

#include <rte_config.h>

#include <rte_atomic.h>
@@ -15,6 +16,8 @@ 
#include "ioat_pci.h"





#include "ioat_pci.h"



/**

 * \file

 *
@@ -64,6 +67,99 @@ ioat_zmalloc(const char *tag, size_t size, unsigned align, uint64_t *phys_addr) 
#define ioat_printf(chan, fmt, args...) printf(fmt, ##args)



#ifdef USE_PCIACCESS



static inline bool

ioat_pci_device_match_id(uint16_t vendor_id, uint16_t device_id)

{

	if (vendor_id != PCI_VENDOR_ID_INTEL) {

		return false;

	}



	switch (device_id) {

	case PCI_DEVICE_ID_INTEL_IOAT_SNB0:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB1:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB2:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB3:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB4:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB5:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB6:

	case PCI_DEVICE_ID_INTEL_IOAT_SNB7:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB0:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB1:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB2:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB3:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB4:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB5:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB6:

	case PCI_DEVICE_ID_INTEL_IOAT_IVB7:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW0:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW1:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW2:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW3:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW4:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW5:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW6:

	case PCI_DEVICE_ID_INTEL_IOAT_HSW7:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX0:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX1:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX2:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX3:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX4:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX5:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX6:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX7:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX8:

	case PCI_DEVICE_ID_INTEL_IOAT_BDX9:

	case PCI_DEVICE_ID_INTEL_IOAT_BWD0:

	case PCI_DEVICE_ID_INTEL_IOAT_BWD1:

	case PCI_DEVICE_ID_INTEL_IOAT_BWD2:

	case PCI_DEVICE_ID_INTEL_IOAT_BWD3:

	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE0:

	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE1:

	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE2:

	case PCI_DEVICE_ID_INTEL_IOAT_BDXDE3:

		return true;

	}



	return false;

}



static inline int

ioat_pci_enumerate(int (*enum_cb)(void *enum_ctx, void *pci_dev), void *enum_ctx)

{

	struct pci_device_iterator *pci_dev_iter;

	struct pci_device *pci_dev;

	struct pci_id_match match;

	int rc;



	match.vendor_id = PCI_VENDOR_ID_INTEL;

	match.subvendor_id = PCI_MATCH_ANY;

	match.subdevice_id = PCI_MATCH_ANY;

	match.device_id = PCI_MATCH_ANY;

	match.device_class = 0x088000;

	match.device_class_mask = 0xFFFFFF;



	pci_dev_iter = pci_id_match_iterator_create(&match);



	rc = 0;

	while ((pci_dev = pci_device_next(pci_dev_iter))) {

		if (!(ioat_pci_device_match_id(pci_dev->vendor_id,

					       pci_dev->device_id))) {

			continue;

		}



		pci_device_probe(pci_dev);



		if (enum_cb(enum_ctx, pci_dev)) {

			rc = -1;

		}

	}



	pci_iterator_destroy(pci_dev_iter);



	return rc;

}



/**

 *

 */