Commit cf0871a5 authored by Daniel Verkamp's avatar Daniel Verkamp Committed by Benjamin Walker
Browse files

ioat: make channel allocation explicit 



Add a parameter to each I/OAT library function that requires a channel
instead of implicitly using the thread-local channel registration model.

I/OAT channels are already reported by the spdk_ioat_probe() attach
callback, so no infrastructure for channel allocation is necessary.

Change-Id: I8731126fcaea9fe2bafc41a3f75c969a100ef8f0
Signed-off-by: default avatarDaniel Verkamp <daniel.verkamp@intel.com>
parent 1dd74730

doc/ioat.index.txt

+0 −1
Original line number Diff line number Diff line
@@ -42,7 +42,6 @@ 
\section key_functions Key Functions



- spdk_ioat_probe() \copybrief spdk_ioat_probe()

- spdk_ioat_register_thread() \copybrief spdk_ioat_register_thread()

- spdk_ioat_get_dma_capabilities() \copybrief spdk_ioat_get_dma_capabilities()

- spdk_ioat_submit_copy() \copybrief spdk_ioat_submit_copy()

- spdk_ioat_submit_fill() \copybrief spdk_ioat_submit_fill()

examples/ioat/perf/perf.c

+30 −10
Original line number Diff line number Diff line
@@ -61,10 +61,12 @@ struct ioat_device { 
};



static TAILQ_HEAD(, ioat_device) g_devices;

static struct ioat_device *g_next_device;



static struct user_config g_user_config;



struct thread_entry {

	struct spdk_ioat_chan *chan;

	uint64_t xfer_completed;

	uint64_t xfer_failed;

	uint64_t current_queue_depth;
@@ -244,7 +246,7 @@ static void 
drain_io(struct thread_entry *thread_entry)

{

	while (thread_entry->current_queue_depth > 0) {

		spdk_ioat_process_events();

		spdk_ioat_process_events(thread_entry->chan);

	}

}
@@ -256,7 +258,8 @@ submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_tas 
	ioat_task->src = src;

	ioat_task->dst = dst;



	spdk_ioat_submit_copy(ioat_task, ioat_done, dst, src, g_user_config.xfer_size_bytes);

	spdk_ioat_submit_copy(thread_entry->chan, ioat_task, ioat_done, dst, src,

			      g_user_config.xfer_size_bytes);



	thread_entry->current_queue_depth++;

}
@@ -283,6 +286,10 @@ work_fn(void *arg) 
	uint64_t tsc_end;

	struct thread_entry *t = (struct thread_entry *)arg;



	if (!t->chan) {

		return 0;

	}



	t->lcore_id = rte_lcore_id();



	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@@ -297,24 +304,17 @@ work_fn(void *arg) 
		return 1;

	}



	if (spdk_ioat_register_thread() != 0) {

		fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",

			rte_lcore_id());

		return 0;

	}



	tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();



	// begin to submit transfers

	submit_xfers(t, g_user_config.queue_depth);

	while (rte_get_timer_cycles() < tsc_end) {

		spdk_ioat_process_events();

		spdk_ioat_process_events(t->chan);

	}



	// begin to drain io

	t->is_draining = true;

	drain_io(t);

	spdk_ioat_unregister_thread();



	return 0;

}
@@ -383,6 +383,23 @@ dump_result(struct thread_entry *threads, int len) 
	return total_failed ? 1 : 0;

}



static struct spdk_ioat_chan *

get_next_chan(void)

{

	struct spdk_ioat_chan *chan;



	if (g_next_device == NULL) {

		fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");

		return NULL;

	}



	chan = g_next_device->ioat;



	g_next_device = TAILQ_NEXT(g_next_device, tailq);



	return chan;

}



int

main(int argc, char **argv)

{
@@ -400,10 +417,13 @@ main(int argc, char **argv) 


	dump_user_config(&g_user_config);



	g_next_device = TAILQ_FIRST(&g_devices);

	RTE_LCORE_FOREACH_SLAVE(lcore_id) {

		threads[lcore_id].chan = get_next_chan();

		rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);

	}



	threads[rte_get_master_lcore()].chan = get_next_chan();

	if (work_fn(&threads[rte_get_master_lcore()]) != 0) {

		rc = 1;

		goto cleanup;

examples/ioat/verify/verify.c

+33 −14
Original line number Diff line number Diff line
@@ -66,10 +66,12 @@ struct ioat_device { 
};



static TAILQ_HEAD(, ioat_device) g_devices;

static struct ioat_device *g_next_device;



static struct user_config g_user_config;



struct thread_entry {

	struct spdk_ioat_chan *chan;

	uint64_t xfer_completed;

	uint64_t xfer_failed;

	uint64_t fill_completed;
@@ -292,7 +294,7 @@ static void 
drain_xfers(struct thread_entry *thread_entry)

{

	while (thread_entry->current_queue_depth > 0) {

		spdk_ioat_process_events();

		spdk_ioat_process_events(thread_entry->chan);

	}

}
@@ -300,10 +302,11 @@ static void 
submit_single_xfer(struct ioat_task *ioat_task)

{

	if (ioat_task->type == IOAT_FILL_TYPE)

		spdk_ioat_submit_fill(ioat_task, ioat_done, ioat_task->dst, ioat_task->fill_pattern,

				      ioat_task->len);

		spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done,

				      ioat_task->dst, ioat_task->fill_pattern, ioat_task->len);

	else

		spdk_ioat_submit_copy(ioat_task, ioat_done, ioat_task->dst, ioat_task->src, ioat_task->len);

		spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done,

				      ioat_task->dst, ioat_task->src, ioat_task->len);

	ioat_task->thread_entry->current_queue_depth++;

}
@@ -316,7 +319,7 @@ submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth) 
		rte_mempool_get(thread_entry->data_pool, &(ioat_task->buffer));



		ioat_task->type = IOAT_COPY_TYPE;

		if (spdk_ioat_get_dma_capabilities() & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {

		if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) {

			if (queue_depth % 2)

				ioat_task->type = IOAT_FILL_TYPE;

		}
@@ -332,6 +335,10 @@ work_fn(void *arg) 
	char buf_pool_name[20], task_pool_name[20];

	struct thread_entry *t = (struct thread_entry *)arg;



	if (!t->chan) {

		return 0;

	}



	t->lcore_id = rte_lcore_id();



	snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id());
@@ -347,24 +354,16 @@ work_fn(void *arg) 
		return 1;

	}



	if (spdk_ioat_register_thread() != 0) {

		fprintf(stderr, "lcore %u: No ioat channels found. Check that ioatdma driver is unloaded.\n",

			rte_lcore_id());

		return 0;

	}



	tsc_end = rte_get_timer_cycles() + g_user_config.time_in_sec * rte_get_timer_hz();



	submit_xfers(t, g_user_config.queue_depth);

	while (rte_get_timer_cycles() < tsc_end) {

		spdk_ioat_process_events();

		spdk_ioat_process_events(t->chan);

	}



	t->is_draining = true;

	drain_xfers(t);



	spdk_ioat_unregister_thread();



	return 0;

}
@@ -437,6 +436,23 @@ dump_result(struct thread_entry *threads, int len) 
	return total_failed ? 1 : 0;

}



static struct spdk_ioat_chan *

get_next_chan(void)

{

	struct spdk_ioat_chan *chan;



	if (g_next_device == NULL) {

		fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n");

		return NULL;

	}



	chan = g_next_device->ioat;



	g_next_device = TAILQ_NEXT(g_next_device, tailq);



	return chan;

}



int

main(int argc, char **argv)

{
@@ -454,10 +470,13 @@ main(int argc, char **argv) 


	dump_user_config(&g_user_config);



	g_next_device = TAILQ_FIRST(&g_devices);

	RTE_LCORE_FOREACH_SLAVE(lcore_id) {

		threads[lcore_id].chan = get_next_chan();

		rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id);

	}



	threads[rte_get_master_lcore()].chan = get_next_chan();

	if (work_fn(&threads[rte_get_master_lcore()]) != 0) {

		rc = 1;

		goto cleanup;

include/spdk/ioat.h

+7 −34
Original line number Diff line number Diff line
@@ -91,50 +91,26 @@ int spdk_ioat_probe(void *cb_ctx, spdk_ioat_probe_cb probe_cb, spdk_ioat_attach_ 
 */

int spdk_ioat_detach(struct spdk_ioat_chan *ioat);



/**

 * Request a DMA engine channel for the calling thread.

 *

 * Must be called before submitting any requests from a thread.

 *

 * The \ref spdk_ioat_unregister_thread() function can be called to release the channel.

 */

int spdk_ioat_register_thread(void);



/**

 * Unregister the current thread's I/OAT channel.

 *

 * This function can be called after \ref spdk_ioat_register_thread() to release the thread's

 * DMA engine channel for use by other threads.

 */

void spdk_ioat_unregister_thread(void);



/**

 * Submit a DMA engine memory copy request.

 *

 * Before submitting any requests on a thread, the thread must be registered

 * using the \ref spdk_ioat_register_thread() function.

 */

int64_t spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,

int64_t spdk_ioat_submit_copy(struct spdk_ioat_chan *chan,

			      void *cb_arg, spdk_ioat_req_cb cb_fn,

			      void *dst, const void *src, uint64_t nbytes);



/**

 * Submit a DMA engine memory fill request.

 *

 * Before submitting any requests on a thread, the thread must be registered

 * using the \ref spdk_ioat_register_thread() function.

 */

int64_t spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,

int64_t spdk_ioat_submit_fill(struct spdk_ioat_chan *chan,

			      void *cb_arg, spdk_ioat_req_cb cb_fn,

			      void *dst, uint64_t fill_pattern, uint64_t nbytes);



/**

 * Check for completed requests on the current thread.

 *

 * Before submitting any requests on a thread, the thread must be registered

 * using the \ref spdk_ioat_register_thread() function.

 * Check for completed requests on an I/OAT channel.

 *

 * \returns 0 on success or negative if something went wrong.

 */

int spdk_ioat_process_events(void);

int spdk_ioat_process_events(struct spdk_ioat_chan *chan);



/**

 * DMA engine capability flags
@@ -146,11 +122,8 @@ enum spdk_ioat_dma_capability_flags { 


/**

 * Get the DMA engine capabilities.

 *

 * Before submitting any requests on a thread, the thread must be registered

 * using the \ref spdk_ioat_register_thread() function.

 */

uint32_t spdk_ioat_get_dma_capabilities(void);

uint32_t spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *chan);



#ifdef __cplusplus

}

lib/ioat/ioat.c

+5 −67
Original line number Diff line number Diff line
@@ -34,15 +34,6 @@ 
#include "ioat_internal.h"

#include "ioat_pci.h"



/** List of channels that have been attached but are not yet assigned to a thread.

 *

 * Must hold g_ioat_driver.lock while manipulating this list.

 */

static SLIST_HEAD(, spdk_ioat_chan) ioat_free_channels;



/** IOAT channel assigned to this thread (or NULL if not assigned yet). */

static __thread struct spdk_ioat_chan *ioat_thread_channel;



struct ioat_driver {

	ioat_mutex_t	lock;

	TAILQ_HEAD(, spdk_ioat_chan)	attached_chans;
@@ -488,8 +479,6 @@ ioat_attach(void *device) 
		return NULL;

	}



	SLIST_INSERT_HEAD(&ioat_free_channels, ioat, next);



	return ioat;

}
@@ -565,7 +554,6 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat) 
	 * when calling ioat_detach().

	 */

	ioat_mutex_lock(&driver->lock);

	SLIST_REMOVE(&ioat_free_channels, ioat, spdk_ioat_chan, next);

	TAILQ_REMOVE(&driver->attached_chans, ioat, tailq);

	ioat_mutex_unlock(&driver->lock);
@@ -575,55 +563,15 @@ spdk_ioat_detach(struct spdk_ioat_chan *ioat) 
	return 0;

}



int

spdk_ioat_register_thread(void)

{

	struct ioat_driver	*driver = &g_ioat_driver;



	if (ioat_thread_channel) {

		ioat_printf(NULL, "%s: thread already registered\n", __func__);

		return -1;

	}



	ioat_mutex_lock(&driver->lock);



	ioat_thread_channel = SLIST_FIRST(&ioat_free_channels);

	if (ioat_thread_channel) {

		SLIST_REMOVE_HEAD(&ioat_free_channels, next);

	}



	ioat_mutex_unlock(&driver->lock);



	return ioat_thread_channel ? 0 : -1;

}



void

spdk_ioat_unregister_thread(void)

{

	struct ioat_driver	*driver = &g_ioat_driver;



	if (!ioat_thread_channel) {

		return;

	}



	ioat_mutex_lock(&driver->lock);



	SLIST_INSERT_HEAD(&ioat_free_channels, ioat_thread_channel, next);

	ioat_thread_channel = NULL;



	ioat_mutex_unlock(&driver->lock);

}



#define min(a, b) (((a)<(b))?(a):(b))



#define _2MB_PAGE(ptr)		((ptr) & ~(0x200000 - 1))

#define _2MB_OFFSET(ptr)	((ptr) &  (0x200000 - 1))



int64_t

spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn,

spdk_ioat_submit_copy(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,

		      void *dst, const void *src, uint64_t nbytes)

{

	struct spdk_ioat_chan	*ioat;

	struct ioat_descriptor	*last_desc;

	uint64_t	remaining, op_size;

	uint64_t	vdst, vsrc;
@@ -631,7 +579,6 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, 
	uint64_t	pdst_page, psrc_page;

	uint32_t	orig_head;



	ioat = ioat_thread_channel;

	if (!ioat) {

		return -1;

	}
@@ -698,16 +645,14 @@ spdk_ioat_submit_copy(void *cb_arg, spdk_ioat_req_cb cb_fn, 
}



int64_t

spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn,

spdk_ioat_submit_fill(struct spdk_ioat_chan *ioat, void *cb_arg, spdk_ioat_req_cb cb_fn,

		      void *dst, uint64_t fill_pattern, uint64_t nbytes)

{

	struct spdk_ioat_chan	*ioat;

	struct ioat_descriptor	*last_desc = NULL;

	uint64_t	remaining, op_size;

	uint64_t	vdst;

	uint32_t	orig_head;



	ioat = ioat_thread_channel;

	if (!ioat) {

		return -1;

	}
@@ -756,11 +701,8 @@ spdk_ioat_submit_fill(void *cb_arg, spdk_ioat_req_cb cb_fn, 
}



uint32_t

spdk_ioat_get_dma_capabilities(void)

spdk_ioat_get_dma_capabilities(struct spdk_ioat_chan *ioat)

{

	struct spdk_ioat_chan	*ioat;



	ioat = ioat_thread_channel;

	if (!ioat) {

		return 0;

	}
@@ -768,11 +710,7 @@ spdk_ioat_get_dma_capabilities(void) 
}



int

spdk_ioat_process_events(void)

spdk_ioat_process_events(struct spdk_ioat_chan *ioat)

{

	if (!ioat_thread_channel) {

		return -1;

	}



	return ioat_process_channel_events(ioat_thread_channel);

	return ioat_process_channel_events(ioat);

}