Commit d58a2f6c authored by paul luse's avatar paul luse Committed by Tomasz Zawadzki
Browse files

lib/accel: support multiple accel modules (aka engines) at once 



We enable multiple engines by:

* getting rid of the globals that point to the one available HW
and one available SW engine

* adding a submit_tasks() entry point for the SW engine so that
it is treated like any other engine allowing us to just call
submit_tasks() to the assigned engine for the opcode instead of
checking what is supported

* changing the definition of engine capabilities from
"HW accelerated" to simply "supported"

* during init, use a global (g_engines_opc) that contains engines
and is indexed by opcode so we know what the best engine is for each
op code

* future patches will add RPC's to override engine priorities or
specifically assign an opcode(s) to an engine.

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: I9b9f3d5a2e499124aa7ccf71f0da83c8ee3dd9f9
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/11870


Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 8f9b9775

include/spdk_internal/accel_engine.h

+4 −4
Original line number Diff line number Diff line
@@ -44,9 +44,7 @@ struct spdk_accel_task; 
void spdk_accel_task_complete(struct spdk_accel_task *task, int status);



struct accel_io_channel {

	struct spdk_accel_engine	*engine;

	struct spdk_io_channel		*engine_ch;

	struct spdk_io_channel		*sw_engine_ch;

	struct spdk_io_channel		*engine_ch[ACCEL_OPC_LAST];

	void				*task_pool_base;

	TAILQ_HEAD(, spdk_accel_task)	task_pool;

};
@@ -85,9 +83,11 @@ struct spdk_accel_task { 
};



struct spdk_accel_engine {

	const char *name;

	bool (*supports_opcode)(enum accel_opcode);

	struct spdk_io_channel *(*get_io_channel)(void);

	int (*submit_tasks)(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task);

	TAILQ_ENTRY(spdk_accel_engine) tailq;

};



struct spdk_accel_module_if {
@@ -118,7 +118,7 @@ struct spdk_accel_module_if { 
	TAILQ_ENTRY(spdk_accel_module_if)	tailq;

};



void spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine);

void spdk_accel_engine_register(struct spdk_accel_engine *accel_engine);

void spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module);



#define SPDK_ACCEL_MODULE_REGISTER(init_fn, fini_fn, config_json, ctx_size_fn)				\

lib/accel/accel_engine.c

+226 −165
Original line number Diff line number Diff line
@@ -60,8 +60,6 @@ 
/* Largest context size for all accel modules */

static size_t g_max_accel_module_size = 0;



static struct spdk_accel_engine *g_hw_accel_engine = NULL;

static struct spdk_accel_engine *g_sw_accel_engine = NULL;

static struct spdk_accel_module_if *g_accel_engine_module = NULL;

static spdk_accel_fini_cb g_fini_cb_fn = NULL;

static void *g_fini_cb_arg = NULL;
@@ -70,46 +68,48 @@ static void *g_fini_cb_arg = NULL; 
static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list =

	TAILQ_HEAD_INITIALIZER(spdk_accel_module_list);



static void _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags);

static void _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags);

static void _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags);

static int _sw_accel_compare(void *src1, void *src2, size_t nbytes);

static void _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags);

static void _sw_accel_crc32c(uint32_t *dst, void *src, uint32_t seed, size_t nbytes);

static void _sw_accel_crc32cv(uint32_t *dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed);

/* Global list of registered engines */

static TAILQ_HEAD(, spdk_accel_engine) g_engine_list =

	TAILQ_HEAD_INITIALIZER(g_engine_list);



/* Registration of hw modules (currently supports only 1 at a time) */

void

spdk_accel_hw_engine_register(struct spdk_accel_engine *accel_engine)

/* Global array mapping capabilities to engines */

static struct spdk_accel_engine *g_engines_opc[ACCEL_OPC_LAST] = {};



static int sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_task);



static struct spdk_accel_engine *

_engine_find_by_name(const char *name)

{

	if (g_hw_accel_engine == NULL) {

		g_hw_accel_engine = accel_engine;

	} else {

		SPDK_NOTICELOG("Hardware offload engine already enabled\n");

	struct spdk_accel_engine *accel_engine = NULL;



	TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {

		if (strcmp(name, accel_engine->name) == 0) {

			break;

		}

	}



/* Registration of sw modules (currently supports only 1) */

static void

accel_sw_register(struct spdk_accel_engine *accel_engine)

{

	assert(g_sw_accel_engine == NULL);

	g_sw_accel_engine = accel_engine;

	return accel_engine;

}



static void

accel_sw_unregister(void)

/* Registration of all engines */

void

spdk_accel_engine_register(struct spdk_accel_engine *engine)

{

	g_sw_accel_engine = NULL;

	if (_engine_find_by_name(engine->name)) {

		SPDK_NOTICELOG("Accel engine %s already registered\n", engine->name);

		assert(false);

		return;

	}



/* Used to determine whether a command is sent to an engine/module or done here

 * via SW implementation.

	/* Make sure that the software engine is at the head of the list, this

	 * will assure that all opcodes are later assigned to software first and

	 * then udpated to HW engines as they are registered.

	 */

inline static bool

_is_supported(struct spdk_accel_engine *engine, enum accel_opcode operation)

{

	return (engine->supports_opcode(operation));

	if (strcmp(engine->name, "software") == 0) {

		TAILQ_INSERT_HEAD(&g_engine_list, engine, tailq);

	} else {

		TAILQ_INSERT_TAIL(&g_engine_list, engine, tailq);

	}

}



void
@@ -152,10 +152,8 @@ _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, voi 
/* Post SW completions to a list and complete in a poller as we don't want to

 * complete them on the caller's stack as they'll likely submit another. */

inline static void

_add_to_comp_list(struct accel_io_channel *accel_ch, struct spdk_accel_task *accel_task, int status)

_add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status)

{

	struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(accel_ch->sw_engine_ch);



	accel_task->status = status;

	TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);

}
@@ -176,12 +174,13 @@ _check_flags(int flags) 


/* Accel framework public API for copy function */

int

spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,

		       int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src,

		       uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY];



	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {
@@ -194,27 +193,19 @@ spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_ 
	accel_task->nbytes = nbytes;

	accel_task->flags = flags;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _check_flags(flags);

		if (rc) {

			return rc;

		}

		_sw_accel_copy(dst, src, (size_t)nbytes, flags);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for dual cast copy function */

int

spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,

			   uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1,

			   void *dst2, void *src, uint64_t nbytes, int flags,

			   spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_DUALCAST];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_DUALCAST];



	if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) {

		SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
@@ -233,27 +224,19 @@ spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, v 
	accel_task->flags = flags;

	accel_task->op_code = ACCEL_OPC_DUALCAST;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_DUALCAST)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _check_flags(flags);

		if (rc) {

			return rc;

		}

		_sw_accel_dualcast(dst1, dst2, src, (size_t)nbytes, flags);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for compare function */

int

spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,

			  spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1,

			  void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,

			  void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COMPARE];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COMPARE];



	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {
@@ -265,23 +248,19 @@ spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, ui 
	accel_task->nbytes = nbytes;

	accel_task->op_code = ACCEL_OPC_COMPARE;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_COMPARE)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _sw_accel_compare(src1, src2, (size_t)nbytes);

		_add_to_comp_list(accel_ch, accel_task, rc);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for fill function */

int

spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,

		       int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst,

		       uint8_t fill, uint64_t nbytes, int flags,

		       spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_FILL];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_FILL];



	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {
@@ -294,26 +273,19 @@ spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint 
	accel_task->flags = flags;

	accel_task->op_code = ACCEL_OPC_FILL;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_FILL)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _check_flags(flags);

		if (rc) {

			return rc;

		}

		_sw_accel_fill(dst, fill, (size_t)nbytes, flags);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for CRC-32C function */

int

spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *src, uint32_t seed,

			 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst,

			 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn,

			 void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C];



	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {
@@ -327,22 +299,19 @@ spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, void *sr 
	accel_task->nbytes = nbytes;

	accel_task->op_code = ACCEL_OPC_CRC32C;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for chained CRC-32C function */

int

spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct iovec *iov,

			  uint32_t iov_cnt, uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst,

			  struct iovec *iov, uint32_t iov_cnt, uint32_t seed,

			  spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch;

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_CRC32C];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_CRC32C];



	if (iov == NULL) {

		SPDK_ERRLOG("iov should not be NULL");
@@ -354,7 +323,6 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct 
		return -EINVAL;

	}



	accel_ch = spdk_io_channel_get_ctx(ch);

	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {

		SPDK_ERRLOG("no memory\n");
@@ -368,24 +336,19 @@ spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, struct 
	accel_task->seed = seed;

	accel_task->op_code = ACCEL_OPC_CRC32C;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_CRC32C)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		_sw_accel_crc32cv(crc_dst, iov, iov_cnt, seed);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for copy with CRC-32C function */

int

spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src,

			      uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, int flags,

			      spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst,

			      void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes,

			      int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C];



	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {
@@ -401,29 +364,19 @@ spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, void *src, 
	accel_task->flags = flags;

	accel_task->op_code = ACCEL_OPC_COPY_CRC32C;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _check_flags(flags);

		if (rc) {

			return rc;

		}

		_sw_accel_copy(dst, src, (size_t)nbytes, flags);

		_sw_accel_crc32c(crc_dst, src, seed, (size_t)nbytes);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Accel framework public API for chained copy + CRC-32C function */

int

spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iovec *src_iovs,

			       uint32_t iov_cnt, uint32_t *crc_dst, uint32_t seed, int flags,

			       spdk_accel_completion_cb cb_fn, void *cb_arg)

spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst,

			       struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst,

			       uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch;

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *accel_task;

	int rc;

	struct spdk_accel_engine *engine = g_engines_opc[ACCEL_OPC_COPY_CRC32C];

	struct spdk_io_channel *engine_ch = accel_ch->engine_ch[ACCEL_OPC_COPY_CRC32C];



	if (src_iovs == NULL) {

		SPDK_ERRLOG("iov should not be NULL");
@@ -435,7 +388,6 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov 
		return -EINVAL;

	}



	accel_ch = spdk_io_channel_get_ctx(ch);

	accel_task = _get_task(accel_ch, cb_fn, cb_arg);

	if (accel_task == NULL) {

		SPDK_ERRLOG("no memory\n");
@@ -451,18 +403,7 @@ spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, struct iov 
	accel_task->flags = flags;

	accel_task->op_code = ACCEL_OPC_COPY_CRC32C;



	if (_is_supported(accel_ch->engine, ACCEL_OPC_COPY_CRC32C)) {

		return accel_ch->engine->submit_tasks(accel_ch->engine_ch, accel_task);

	} else {

		rc = _check_flags(flags);

		if (rc) {

			return rc;

		}

		_sw_accel_copyv(dst, src_iovs, iov_cnt, flags);

		_sw_accel_crc32cv(crc_dst, src_iovs, iov_cnt, seed);

		_add_to_comp_list(accel_ch, accel_task, 0);

		return 0;

	}

	return engine->submit_tasks(engine_ch, accel_task);

}



/* Helper function when when accel modules register with the framework. */
@@ -481,7 +422,7 @@ accel_engine_create_cb(void *io_device, void *ctx_buf) 
	struct accel_io_channel	*accel_ch = ctx_buf;

	struct spdk_accel_task *accel_task;

	uint8_t *task_mem;

	int i;

	int i, j;



	accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size);

	if (accel_ch->task_pool_base == NULL) {
@@ -496,21 +437,21 @@ accel_engine_create_cb(void *io_device, void *ctx_buf) 
		task_mem += g_max_accel_module_size;

	}



	/* Set sw engine channel for operations where hw engine does not support. */

	accel_ch->sw_engine_ch = g_sw_accel_engine->get_io_channel();

	assert(accel_ch->sw_engine_ch != NULL);



	if (g_hw_accel_engine != NULL) {

		accel_ch->engine_ch = g_hw_accel_engine->get_io_channel();

		accel_ch->engine = g_hw_accel_engine;

	} else {

		/* No hw engine enabled, use sw. */

		accel_ch->engine_ch = accel_ch->sw_engine_ch;

		accel_ch->engine = g_sw_accel_engine;

	/* Assign engines and get IO channels for each */

	for (i = 0; i < ACCEL_OPC_LAST; i++) {

		accel_ch->engine_ch[i] = g_engines_opc[i]->get_io_channel();

		/* This can happen if idxd runs out of channels. */

		if (accel_ch->engine_ch[i] == NULL) {

			goto err;

		}

	}

	assert(accel_ch->engine_ch != NULL);



	return 0;

err:

	for (j = 0; j < i; j++) {

		spdk_put_io_channel(accel_ch->engine_ch[j]);

	}

	return -EINVAL;

}



/* Framework level channel destroy callback. */
@@ -518,11 +459,15 @@ static void 
accel_engine_destroy_cb(void *io_device, void *ctx_buf)

{

	struct accel_io_channel	*accel_ch = ctx_buf;

	int i;



	if (accel_ch->sw_engine_ch != accel_ch->engine_ch) {

		spdk_put_io_channel(accel_ch->sw_engine_ch);

	for (i = 0; i < ACCEL_OPC_LAST; i++) {

		if (accel_ch->engine_ch[i]) {

			spdk_put_io_channel(accel_ch->engine_ch[i]);

			accel_ch->engine_ch[i] = NULL;

		}

	spdk_put_io_channel(accel_ch->engine_ch);

	}



	free(accel_ch->task_pool_base);

}
@@ -545,9 +490,30 @@ accel_engine_module_initialize(void) 
int

spdk_accel_engine_initialize(void)

{

	SPDK_NOTICELOG("Accel engine initialized to use software engine.\n");

	enum accel_opcode op;

	struct spdk_accel_engine *accel_engine = NULL;



	accel_engine_module_initialize();



	/* Create our priority global map of opcodes to engines, we populate starting

	 * with the software engine (guaranteed to be first on the list) and then

	 * updating opcodes with HW engines that have been initilaized.

	 * NOTE: all opcodes must be suported by software in the event that no HW

	 * engines are initilaized to support the operation.

	 */

	TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {

		for (op = 0; op < ACCEL_OPC_LAST; op++) {

			if (accel_engine->supports_opcode(op)) {

				g_engines_opc[op] = accel_engine;

				SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_engine->name);

			}

		}

	}

#ifdef DEBUG

	for (op = 0; op < ACCEL_OPC_LAST; op++) {

		assert(g_engines_opc[op] != NULL);

	}

#endif

	/*

	 * We need a unique identifier for the accel engine framework, so use the

	 * spdk_accel_module_list address for this purpose.
@@ -625,8 +591,18 @@ spdk_accel_engine_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 
static bool

sw_accel_supports_opcode(enum accel_opcode opc)

{

	switch (opc) {

	case ACCEL_OPC_COPY:

	case ACCEL_OPC_FILL:

	case ACCEL_OPC_DUALCAST:

	case ACCEL_OPC_COMPARE:

	case ACCEL_OPC_CRC32C:

	case ACCEL_OPC_COPY_CRC32C:

		return true;

	default:

		return false;

	}

}



static inline void

_pmem_memcpy(void *dst, const void *src, size_t len)
@@ -725,12 +701,87 @@ _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_ 
	*crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed);

}



static struct spdk_io_channel *sw_accel_get_io_channel(void);

static int

sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task)

{

	struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *tmp;

	int rc = 0;



	do {

		switch (accel_task->op_code) {

		case ACCEL_OPC_COPY:

			rc = _check_flags(accel_task->flags);

			if (rc == 0) {

				_sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags);

			}

			_add_to_comp_list(sw_ch, accel_task, rc);

			break;

		case ACCEL_OPC_FILL:

			rc = _check_flags(accel_task->flags);

			if (rc == 0) {

				_sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags);

			}

			_add_to_comp_list(sw_ch, accel_task, rc);

			break;

		case ACCEL_OPC_DUALCAST:

			rc = _check_flags(accel_task->flags);

			if (rc == 0) {

				_sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes,

						   accel_task->flags);

			}

			_add_to_comp_list(sw_ch, accel_task, rc);

			break;

		case ACCEL_OPC_COMPARE:

			rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes);

			_add_to_comp_list(sw_ch, accel_task, rc);

			break;

		case ACCEL_OPC_CRC32C:

			if (accel_task->v.iovcnt == 0) {

				_sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes);

			} else {

				_sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed);

			}

			_add_to_comp_list(sw_ch, accel_task, 0);

			break;

		case ACCEL_OPC_COPY_CRC32C:

			rc = _check_flags(accel_task->flags);

			if (rc == 0) {

				if (accel_task->v.iovcnt == 0) {

					_sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags);

					_sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes);

				} else {

					_sw_accel_copyv(accel_task->dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->flags);

					_sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed);

				}

			}

			_add_to_comp_list(sw_ch, accel_task, rc);

			break;

		default:

			assert(false);

			break;

		}



		tmp = TAILQ_NEXT(accel_task, link);



		/* Report any build errors via the callback now. */

		if (rc) {

			spdk_accel_task_complete(accel_task, rc);

		}



		accel_task = tmp;

	} while (accel_task);



	return 0;

}



static struct spdk_io_channel *sw_accel_get_io_channel(void);



static struct spdk_accel_engine sw_accel_engine = {

	.name			= "software",

	.supports_opcode	= sw_accel_supports_opcode,

	.get_io_channel		= sw_accel_get_io_channel,

	.submit_tasks		= sw_accel_submit_tasks,

};



static int
@@ -788,7 +839,8 @@ sw_accel_engine_get_ctx_size(void) 
static int

sw_accel_engine_init(void)

{

	accel_sw_register(&sw_accel_engine);

	SPDK_NOTICELOG("Accel framework software engine initialized.\n");

	spdk_accel_engine_register(&sw_accel_engine);

	spdk_io_device_register(&sw_accel_engine, sw_accel_create_cb, sw_accel_destroy_cb,

				sizeof(struct sw_accel_io_channel), "sw_accel_engine");
@@ -798,8 +850,17 @@ sw_accel_engine_init(void) 
static void

sw_accel_engine_fini(void *ctxt)

{

	struct spdk_accel_engine *accel_engine;



	spdk_io_device_unregister(&sw_accel_engine, NULL);

	accel_sw_unregister();



	/* unregister the software engine */

	TAILQ_FOREACH(accel_engine, &g_engine_list, tailq) {

		if (strcmp(accel_engine->name, "software") == 0) {

			TAILQ_REMOVE(&g_engine_list, accel_engine, tailq);

			break;

		}

	}



	spdk_accel_engine_module_finish();

}

lib/accel/spdk_accel.map

+1 −1
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ 
	spdk_accel_write_config_json;



	# functions needed by modules

	spdk_accel_hw_engine_register;

	spdk_accel_engine_register;

	spdk_accel_module_list_add;

	spdk_accel_task_complete;

module/accel/idxd/accel_engine_idxd.c

+7 −4
Original line number Diff line number Diff line
@@ -130,7 +130,9 @@ static void 
idxd_done(void *cb_arg, int status)

{

	struct spdk_accel_task *accel_task = cb_arg;

	struct idxd_io_channel *chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch);

	struct idxd_io_channel *chan;



	chan = spdk_io_channel_get_ctx(accel_task->accel_ch->engine_ch[accel_task->op_code]);



	assert(chan->num_outstanding > 0);

	spdk_trace_record(TRACE_IDXD_OP_COMPLETE, 0, 0, 0, chan->num_outstanding - 1);
@@ -302,7 +304,7 @@ idxd_poll(void *arg) 


			TAILQ_INIT(&chan->queued_tasks);



			idxd_submit_tasks(task->accel_ch->engine_ch, task);

			idxd_submit_tasks(task->accel_ch->engine_ch[task->op_code], task);

		}

	}
@@ -332,6 +334,7 @@ idxd_supports_opcode(enum accel_opcode opc) 
}



static struct spdk_accel_engine idxd_accel_engine = {

	.name			= "idxd",

	.supports_opcode	= idxd_supports_opcode,

	.get_io_channel		= idxd_get_io_channel,

	.submit_tasks		= idxd_submit_tasks,
@@ -419,8 +422,8 @@ accel_engine_idxd_init(void) 
	}



	g_idxd_initialized = true;

	SPDK_NOTICELOG("Accel engine updated to use IDXD DSA engine.\n");

	spdk_accel_hw_engine_register(&idxd_accel_engine);

	SPDK_NOTICELOG("Accel framework IDXD engine initialized.\n");

	spdk_accel_engine_register(&idxd_accel_engine);

	spdk_io_device_register(&idxd_accel_engine, idxd_create_cb, idxd_destroy_cb,

				sizeof(struct idxd_io_channel), "idxd_accel_engine");

	return 0;

module/accel/ioat/accel_engine_ioat.c

+3 −2
Original line number Diff line number Diff line
@@ -182,6 +182,7 @@ ioat_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task 
}



static struct spdk_accel_engine ioat_accel_engine = {

	.name			= "ioat",

	.supports_opcode	= ioat_supports_opcode,

	.get_io_channel		= ioat_get_io_channel,

	.submit_tasks		= ioat_submit_tasks,
@@ -289,8 +290,8 @@ accel_engine_ioat_init(void) 
	}



	g_ioat_initialized = true;

	SPDK_NOTICELOG("Accel engine updated to use IOAT engine.\n");

	spdk_accel_hw_engine_register(&ioat_accel_engine);

	SPDK_NOTICELOG("Accel framework IOAT engine initialized.\n");

	spdk_accel_engine_register(&ioat_accel_engine);

	spdk_io_device_register(&ioat_accel_engine, ioat_create_cb, ioat_destroy_cb,

				sizeof(struct ioat_io_channel), "ioat_accel_engine");

	return 0;