Commit fc250841 authored by paul luse's avatar paul luse Committed by Tomasz Zawadzki
Browse files

idxd: add batch capability to accel framework and IDXD back-end 



This patch only includes the basic framework for batching and the
ability to batch one type of command, copy. Follow-on patches will
add the ability to batch other commands and include an example of
how to do so via the accel perf tool.  SW engine support for batching
will also come in a future patch. Documentation will also be coming.

Batching allows the application to submit a list of independent
descriptors to DSA with one single "batch" descriptor. This is beneficial
when the application is in a position to have several operations ready
at once; batching saves the overhead of submitting each one separately.

The way batching works in SPDK is as follows:

1) The app gets a handle to a new batch with spdk_accel_batch_create()
2) The app uses that handle to prepare a command to be included in the
batch. For copy the command is spdk_accel_batch_prep_copy(). The
app many continue to prep commands for the batch up to the max via
calling spdk_accel_batch_get_max()
3) The app then submits the batch with spdk_accel_batch_submit()
4) The callback provided for each command in the batch will be called as
they complete, the callback provided to the batch submit itself will be
called then the entire batch is done.

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: I4102e9291fe59a245cedde6888f42a923b6dbafd
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2248


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent cf83cac4

doc/idxd.md

+4 −0
Original line number Diff line number Diff line
@@ -9,6 +9,10 @@ 
Function                                | Description

--------------------------------------- | -----------

spdk_idxd_probe()                       | @copybrief spdk_idxd_probe()

spdk_idxd_batch_get_max()               | @copybrief spdk_idxd_batch_get_max()

spdk_idxd_batch_create()                | @copybrief spdk_idxd_batch_create()

spdk_idxd_batch_prep_copy()             | @copybrief spdk_idxd_batch_prep_copy()

spdk_idxd_batch_submit()                | @copybrief spdk_idxd_batch_submit()

spdk_idxd_submit_copy()                 | @copybrief spdk_idxd_submit_copy()

spdk_idxd_submit_compare()              | @copybrief spdk_idxd_submit_compare()

spdk_idxd_submit_crc32c()               | @copybrief spdk_idxd_submit_crc32c()

include/spdk/accel_engine.h

+61 −12
Original line number Diff line number Diff line
@@ -73,6 +73,8 @@ struct spdk_io_channel; 


struct spdk_accel_task;



struct spdk_accel_batch;



/**

 * Initialize the acceleration engine.

 *
@@ -112,8 +114,7 @@ struct spdk_io_channel *spdk_accel_engine_get_io_channel(void); 
/**

 * Retrieve accel engine capabilities.

 *

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel associated with this call.

 *

 * \return bitmap of capabilities defined by enum accel_capability.

 */
@@ -123,8 +124,7 @@ uint64_t spdk_accel_get_capabilities(struct spdk_io_channel *ch); 
 * Submit a copy request.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel to submit request to the accel engine.

 * \param dst Destination to copy to.

 * \param src Source to copy from.

 * \param nbytes Length in bytes to copy.
@@ -135,12 +135,64 @@ uint64_t spdk_accel_get_capabilities(struct spdk_io_channel *ch); 
int spdk_accel_submit_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, void *dst,

			   void *src, uint64_t nbytes, spdk_accel_completion_cb cb);



/**

 * Synchronous call to get batch size.  This is the maximum number of

 *  descriptors that a batch can contain. Once this limit is reached the batch

 *  should be processed with spdk_accel_batch_submit().

 *

 * \param ch I/O channel associated with this call.

 *

 * \return max number of descriptors per batch.

 */

uint32_t spdk_accel_batch_get_max(struct spdk_io_channel *ch);



/**

 * Synchronous call to create a batch sequence.

 *

 * \param ch I/O channel associated with this call.

 *

 * \return handle to use for subsequent batch requests, NULL on failure.

 */

struct spdk_accel_batch *spdk_accel_batch_create(struct spdk_io_channel *ch);



/**

 * Asynchronous call to submit a batch sequence.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel associated with this call.

 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().

 * \param cb Called when this operation completes.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

			    struct spdk_accel_batch *batch, spdk_accel_completion_cb cb);



/**

 * Synchronous call to prepare a copy request into a previously initialized batch

 *  created with spdk_accel_batch_create(). The callback will be called when the copy

 *  completes after the batch has been submitted by an asynchronous call to

 *  spdk_accel_batch_submit().

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel associated with this call.

 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().

 * \param dst Destination to copy to.

 * \param src Source to copy from.

 * \param nbytes Length in bytes to copy.

 * \param cb Called when this operation completes.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

			       struct spdk_accel_batch *batch, void *dst, void *src,

			       uint64_t nbytes, spdk_accel_completion_cb cb);



/**

 * Submit a dual cast copy request.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel to submit request to the accel engine.

 * \param dst1 First destination to copy to (must be 4K aligned).

 * \param dst2 Second destination to copy to (must be 4K aligned).

 * \param src Source to copy from.
@@ -157,8 +209,7 @@ int spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io 
 * Submit a compare request.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel to submit request to the accel engine.

 * \param src1 First location to perform compare on.

 * \param src2 Second location to perform compare on.

 * \param nbytes Length in bytes to compare.
@@ -176,8 +227,7 @@ int spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_ 
 * This operation will fill the destination buffer with the specified value.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel to submit request to the accel engine.

 * \param dst Destination to fill.

 * \param fill Constant byte to fill to the destination.

 * \param nbytes Length in bytes to fill.
@@ -194,8 +244,7 @@ int spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_cha 
 * This operation will calculate the 4 byte CRC32-C for the given data.

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel to submit request to the accel engine. This channel can

 * be obtained by the function spdk_accel_engine_get_io_channel().

 * \param ch I/O channel to submit request to the accel engine.

 * \param dst Destination to write the CRC-32C to.

 * \param src The source address for the data.

 * \param seed Four byte seed value.

include/spdk/idxd.h

+60 −5
Original line number Diff line number Diff line
@@ -56,6 +56,11 @@ struct spdk_idxd_io_channel; 
 */

struct spdk_idxd_device;



/**

 * Opaque handle for batching.

 */

struct idxd_batch;



/**

 * Signature for configuring a channel

 *
@@ -138,7 +143,57 @@ void spdk_idxd_detach(struct spdk_idxd_device *idxd); 
void spdk_idxd_set_config(uint32_t config_number);



/**

 * Build and submit a accel engine memory copy request.

 * Return the max number of descriptors per batch for IDXD.

 *

 * \return max number of desciptors per batch.

 */

uint32_t spdk_idxd_batch_get_max(void);



/**

 * Create a batch sequence.

 *

 * \param chan IDXD channel to submit request.

 *

 * \return handle to use for subsequent batch requests, NULL on failure.

 */

struct idxd_batch *spdk_idxd_batch_create(struct spdk_idxd_io_channel *chan);



/**

 * Submit a batch sequence.

 *

 * \param chan IDXD channel to submit request.

 * \param batch Handle provided when the batch was started with spdk_idxd_batch_create().

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the arg parameter in

 * the completion callback.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,

			   spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Synchronous call to prepare a copy request into a previously initialized batch

 *  created with spdk_idxd_batch_create(). The callback will be called when the copy

 *  completes after the batch has been submitted by an asynchronous call to

 *  spdk_idxd_batch_submit().

 *

 * \param chan IDXD channel to submit request.

 * \param batch Handle provided when the batch was started with spdk_idxd_batch_create().

 * \param dst Destination virtual address.

 * \param src Source virtual address.

 * \param nbytes Number of bytes to copy.

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the arg parameter in

 * the completion callback.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch,

			      void *dst, const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit a idxd memory copy request.

 *

 * This function will build the copy descriptor and then immediately submit

 * by writing to the proper device portal.
@@ -158,7 +213,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, 
			  spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit an accel engine dual cast copy request.

 * Build and submit an idxd dual cast copy request.

 *

 * This function will build the dual cast descriptor and then immediately submit

 * by writing to the proper device portal.
@@ -179,7 +234,7 @@ int spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, 
			      spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit a memory compare request.

 * Build and submit an idxd memory compare request.

 *

 * This function will build the compare descriptor and then immediately submit

 * by writing to the proper device portal.
@@ -199,7 +254,7 @@ int spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, 
			     spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit a accel engine memory fill request.

 * Build and submit an idxd memory fill request.

 *

 * This function will build the fill descriptor and then immediately submit

 * by writing to the proper device portal.
@@ -219,7 +274,7 @@ int spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, 
			  spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit a memory CRC32-C request.

 * Build and submit an idxd memory CRC32-C request.

 *

 * This function will build the CRC-32C descriptor and then immediately submit

 * by writing to the proper device portal.

include/spdk_internal/accel_engine.h

+6 −0
Original line number Diff line number Diff line
@@ -50,6 +50,12 @@ struct spdk_accel_engine { 
		    uint64_t nbytes, spdk_accel_completion_cb cb);

	int (*dualcast)(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,

			uint64_t nbytes, spdk_accel_completion_cb cb);

	uint32_t (*batch_get_max)(void);

	struct spdk_accel_batch *(*batch_create)(struct spdk_io_channel *ch);

	int (*batch_prep_copy)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			       void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb);

	int (*batch_submit)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			    spdk_accel_completion_cb cb);

	int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2,

		       uint64_t nbytes, spdk_accel_completion_cb cb);

	int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill,

lib/accel/accel_engine.c

+48 −0
Original line number Diff line number Diff line
@@ -140,6 +140,50 @@ spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_cha 
					  _accel_engine_done);

}



/* Accel framework public API for batch_create function */

struct spdk_accel_batch *

spdk_accel_batch_create(struct spdk_io_channel *ch)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);



	return accel_ch->engine->batch_create(accel_ch->ch);

}



/* Accel framework public API for batch_submit function */

int

spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

			struct spdk_accel_batch *batch, spdk_accel_completion_cb cb)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);



	accel_req->cb = cb;

	return accel_ch->engine->batch_submit(accel_req->offload_ctx, accel_ch->ch, batch,

					      _accel_engine_done);

}



/* Accel framework public API for getting max batch */

uint32_t

spdk_accel_batch_get_max(struct spdk_io_channel *ch)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);



	return accel_ch->engine->batch_get_max();

}



/* Accel framework public API for batch prep_copy function */

int

spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

			   struct spdk_accel_batch *batch, void *dst, void *src, uint64_t nbytes,

			   spdk_accel_completion_cb cb)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);



	accel_req->cb = cb;

	return accel_ch->engine->batch_prep_copy(accel_req->offload_ctx, accel_ch->ch, batch, dst, src,

			nbytes,

			_accel_engine_done);

}



/* Accel framework public API for compare function */

int

spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
@@ -419,6 +463,10 @@ static struct spdk_accel_engine sw_accel_engine = { 
	.get_capabilities	= sw_accel_get_capabilities,

	.copy			= sw_accel_submit_copy,

	.dualcast		= sw_accel_submit_dualcast,

	.batch_get_max		= NULL, /* TODO */

	.batch_create		= NULL, /* TODO */

	.batch_prep_copy	= NULL, /* TODO */

	.batch_submit		= NULL, /* TODO */

	.compare		= sw_accel_submit_compare,

	.fill			= sw_accel_submit_fill,

	.crc32c			= sw_accel_submit_crc32c,