Commit ec086e6f authored by paul luse's avatar paul luse Committed by Tomasz Zawadzki
Browse files

lib/accel: add support for batch dualcast to fw and sw engine 



Includes a few related changes:

* slight refactor to have common prep function for batch prep
commands in the sw engine
* added support for dualcast batching in accel_perf example app

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: If9f078a0f8eb509d296e60a94331c92a72f8cb10
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/2913


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 90c4ae45

examples/accel/perf/accel_perf.c

+8 −5
Original line number Diff line number Diff line
@@ -461,6 +461,11 @@ _batch_prep_cmd(struct worker_thread *worker, struct ap_task *task, struct spdk_ 
						worker->ch, batch, task->dst,

						task->src, g_xfer_size_bytes, accel_done);

		break;

	case ACCEL_DUALCAST:

		rc = spdk_accel_batch_prep_dualcast(__accel_task_from_ap_task(task),

						    worker->ch, batch, task->dst, task->dst2,

						    task->src, g_xfer_size_bytes, accel_done);

		break;

	default:

		assert(false);

		break;
@@ -511,11 +516,9 @@ _init_thread(void *arg1) 
	g_num_workers++;

	pthread_mutex_unlock(&g_workers_lock);



	/* TODO: remove the check for ACCEL_COPY as other workloads are enabled for

	 * batching. It's a lot of code per workload so they are beeing added in

	 * separate patches.

	 */

	if (g_workload_selection == ACCEL_COPY && ((g_capabilites & ACCEL_BATCH) == ACCEL_BATCH) &&

	/* TODO: remove the workload selection checks once all are added. */

	if ((g_workload_selection == ACCEL_COPY || g_workload_selection == ACCEL_DUALCAST)

	    && ((g_capabilites & ACCEL_BATCH) == ACCEL_BATCH) &&

	    g_queue_depth > 1) {



		/* Selected engine supports batching and we have enough, so do it. */

include/spdk/accel_engine.h

+21 −0
Original line number Diff line number Diff line
@@ -188,6 +188,27 @@ int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io 
			       struct spdk_accel_batch *batch, void *dst, void *src,

			       uint64_t nbytes, spdk_accel_completion_cb cb);



/**

 * Synchronous call to prepare a dualcast request into a previously initialized batch

 *  created with spdk_accel_batch_create(). The callback will be called when the dualcast

 *  completes after the batch has been submitted by an asynchronous call to

 *  spdk_accel_batch_submit().

 *

 * \param accel_req Accel request task.

 * \param ch I/O channel associated with this call.

 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().

 * \param dst1 First destination to copy to (must be 4K aligned).

 * \param dst2 Second destination to copy to (must be 4K aligned).

 * \param src Source to copy from.

 * \param nbytes Length in bytes to copy.

 * \param cb Called when this operation completes.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_batch_prep_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

				   struct spdk_accel_batch *batch, void *dst1, void *dst2, void *src,

				   uint64_t nbytes, spdk_accel_completion_cb cb);



/**

 * Submit a dual cast copy request.

 *

include/spdk_internal/accel_engine.h

+2 −0
Original line number Diff line number Diff line
@@ -54,6 +54,8 @@ struct spdk_accel_engine { 
	struct spdk_accel_batch *(*batch_create)(struct spdk_io_channel *ch);

	int (*batch_prep_copy)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			       void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb);

	int (*batch_prep_dualcast)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

				   void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb);

	int (*batch_submit)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			    spdk_accel_completion_cb cb);

	int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2,

lib/accel/accel_engine.c

+69 −6
Original line number Diff line number Diff line
@@ -184,6 +184,24 @@ spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_cha 
			_accel_engine_done);

}



/* Accel framework public API for batch prep_dualcast function */

int

spdk_accel_batch_prep_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,

			       struct spdk_accel_batch *batch, void *dst1, void *dst2, void *src, uint64_t nbytes,

			       spdk_accel_completion_cb cb)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);



	if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {

		SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");

		return -EINVAL;

	}



	accel_req->cb = cb;

	return accel_ch->engine->batch_prep_dualcast(accel_req->offload_ctx, accel_ch->ch,

			batch, dst1, dst2, src,	nbytes, _accel_engine_done);

}



/* Accel framework public API for compare function */

int

spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
@@ -440,16 +458,15 @@ sw_accel_batch_start(struct spdk_io_channel *ch) 
	return (struct spdk_accel_batch *)&sw_ch->batch;

}



static int

sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			 void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)

static struct sw_accel_op *

_prep_op(void *cb_arg, struct sw_accel_io_channel *sw_ch, struct spdk_accel_batch *batch,

	 spdk_accel_completion_cb cb)

{

	struct sw_accel_op *op;

	struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);



	if ((struct spdk_accel_batch *)&sw_ch->batch != batch) {

		SPDK_ERRLOG("Invalid batch\n");

		return -EINVAL;

		return NULL;

	}



	if (!TAILQ_EMPTY(&sw_ch->op_pool)) {
@@ -457,12 +474,29 @@ sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_a 
		TAILQ_REMOVE(&sw_ch->op_pool, op, link);

	} else {

		SPDK_ERRLOG("Ran out of operations for batch\n");

		return -ENOMEM;

		return NULL;

	}



	op->cb_arg = cb_arg;

	op->cb_fn = cb;

	op->sw_ch = sw_ch;



	return op;

}



static int

sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

			 void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)

{

	struct sw_accel_op *op;

	struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);



	op = _prep_op(cb_arg, sw_ch, batch, cb);

	if (op == NULL) {

		return -EINVAL;

	}



	/* Command specific. */

	op->src = src;

	op->dst = dst;

	op->nbytes = nbytes;
@@ -472,6 +506,30 @@ sw_accel_batch_prep_copy(void *cb_arg, struct spdk_io_channel *ch, struct spdk_a 
	return 0;

}



static int

sw_accel_batch_prep_dualcast(void *cb_arg, struct spdk_io_channel *ch,

			     struct spdk_accel_batch *batch,

			     void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)

{

	struct sw_accel_op *op;

	struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);



	op = _prep_op(cb_arg, sw_ch, batch, cb);

	if (op == NULL) {

		return -EINVAL;

	}



	/* Command specific. */

	op->src = src;

	op->dst = dst1;

	op->dst2 = dst2;

	op->nbytes = nbytes;

	op->op_code = SW_ACCEL_OPCODE_DUALCAST;

	TAILQ_INSERT_TAIL(&sw_ch->batch, op, link);



	return 0;

}



static int

sw_accel_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,

		      spdk_accel_completion_cb cb)
@@ -495,6 +553,10 @@ sw_accel_batch_submit(void *cb_arg, struct spdk_io_channel *ch, struct spdk_acce 
		case SW_ACCEL_OPCODE_MEMMOVE:

			memcpy(op->dst, op->src, op->nbytes);

			break;

		case SW_ACCEL_OPCODE_DUALCAST:

			memcpy(op->dst, op->src, op->nbytes);

			memcpy(op->dst2, op->src, op->nbytes);

			break;

		default:

			assert(false);

			break;
@@ -598,6 +660,7 @@ static struct spdk_accel_engine sw_accel_engine = { 
	.batch_get_max		= sw_accel_batch_get_max,

	.batch_create		= sw_accel_batch_start,

	.batch_prep_copy	= sw_accel_batch_prep_copy,

	.batch_prep_dualcast	= sw_accel_batch_prep_dualcast,

	.batch_submit		= sw_accel_batch_submit,

	.compare		= sw_accel_submit_compare,

	.fill			= sw_accel_submit_fill,

lib/accel/spdk_accel.map

+1 −0
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 
	spdk_accel_batch_get_max;

	spdk_accel_batch_create;

	spdk_accel_batch_prep_copy;

	spdk_accel_batch_prep_dualcast;

	spdk_accel_batch_submit;

	spdk_accel_submit_copy;

	spdk_accel_submit_dualcast;