lib/idxd: add support for batching dualcast commands (22058d70) · Commits · Public Repositories / spdk

include/spdk/idxd.h

+24 −3

Original line number	Diff line number	Diff line
		@@ -32,7 +32,7 @@
		*/

		/** \file
		* IDXD accel engine driver public interface
		* IDXD driver public interface
		*/

		#ifndef SPDK_IDXD_H
		@@ -193,7 +193,28 @@ int spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel *chan, struct idxd_bat
		void dst, const void src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg);

		/**
		* Build and submit a idxd memory copy request.
		* Synchronous call to prepare a dualcast request into a previously initialized batch
		* created with spdk_idxd_batch_create(). The callback will be called when the dualcast
		* completes after the batch has been submitted by an asynchronous call to
		* spdk_idxd_batch_submit().
		*
		* \param chan IDXD channel to submit request.
		* \param batch Handle provided when the batch was started with spdk_idxd_batch_create().
		* \param dst1 First destination virtual address (must be 4K aligned).
		* \param dst2 Second destination virtual address (must be 4K aligned).
		* \param src Source virtual address.
		* \param nbytes Number of bytes to copy.
		* \param cb_fn Callback function which will be called when the request is complete.
		* \param cb_arg Opaque value which will be passed back as the arg parameter in
		* the completion callback.
		*
		* \return 0 on success, negative errno on failure.
		*/
		int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel chan, struct idxd_batch batch,
		void dst1, void dst2, const void src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void cb_arg);

		/**
		* Build and submit an idxd memory copy request.
		*
		* This function will build the copy descriptor and then immediately submit
		* by writing to the proper device portal.
		@@ -213,7 +234,7 @@ int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,
		spdk_idxd_req_cb cb_fn, void *cb_arg);

		/**
		* Build and submit an idxd dual cast copy request.
		* Build and submit an idxd dualcast request.
		*
		* This function will build the dual cast descriptor and then immediately submit
		* by writing to the proper device portal.

lib/idxd/idxd.c

+45 −0

Original line number	Diff line number	Diff line
		@@ -1005,6 +1005,50 @@ spdk_idxd_batch_prep_copy(struct spdk_idxd_io_channel chan, struct idxd_batch
		return 0;
		}

		int
		spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel chan, struct idxd_batch batch,
		void dst1, void dst2, const void src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void cb_arg)
		{
		struct idxd_hw_desc *desc;
		struct idxd_comp *comp;

		if ((uintptr_t)dst1 & (ALIGN_4K - 1) \|\| (uintptr_t)dst2 & (ALIGN_4K - 1)) {
		SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n");
		return -EINVAL;
		}

		if (_does_batch_exist(batch, chan) == false) {
		SPDK_ERRLOG("Attempt to add to a batch that doesn't exist\n.");
		return -EINVAL;
		}

		if ((batch->cur_index - batch->start_index) == DESC_PER_BATCH) {
		SPDK_ERRLOG("Attempt to add to a batch that is already full\n.");
		return -ENOMEM;
		}

		desc = &chan->ring_ctrl.user_desc[batch->cur_index];
		comp = &chan->ring_ctrl.user_completions[batch->cur_index];
		SPDK_DEBUGLOG(SPDK_LOG_IDXD, "Prep batch %p index %u\n", batch, batch->cur_index);

		batch->cur_index++;
		assert(batch->cur_index > batch->start_index);

		desc->flags = IDXD_FLAG_COMPLETION_ADDR_VALID \| IDXD_FLAG_REQUEST_COMPLETION;
		desc->opcode = IDXD_OPCODE_DUALCAST;
		desc->src_addr = (uint64_t)src;
		desc->dst_addr = (uint64_t)dst1;
		desc->dest2 = (uint64_t)dst2;
		desc->xfer_size = nbytes;

		desc->completion_addr = (uint64_t)&comp->hw;
		comp->cb_arg = cb_arg;
		comp->cb_fn = cb_fn;
		comp->batch = batch;

		return 0;
		}

		static void
		_dump_error_reg(struct spdk_idxd_io_channel *chan)
		{
		@@ -1070,6 +1114,7 @@ _spdk_idxd_process_batch_events(struct spdk_idxd_io_channel *chan)
		status = comp->hw.result;
		}
		break;
		case IDXD_OPCODE_DUALCAST:
		case IDXD_OPCODE_MEMMOVE:
		break;
		default:

lib/idxd/spdk_idxd.map

+1 −0

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@
		spdk_idxd_probe;
		spdk_idxd_detach;
		spdk_idxd_batch_prep_copy;
		spdk_idxd_batch_prep_dualcast;
		spdk_idxd_batch_submit;
		spdk_idxd_batch_create;
		spdk_idxd_batch_get_max;

module/accel/idxd/accel_engine_idxd.c

+17 −0

Original line number	Diff line number	Diff line
		@@ -47,6 +47,8 @@
		#include "spdk/util.h"
		#include "spdk/json.h"

		#define ALIGN_4K 0x1000

		static bool g_idxd_enable = false;
		uint32_t g_config_number;

		@@ -494,12 +496,27 @@ idxd_batch_prep_copy(void cb_arg, struct spdk_io_channel ch, struct spdk_accel
		idxd_done, idxd_task);
		}

		static int
		idxd_batch_prep_dualcast(void cb_arg, struct spdk_io_channel ch, struct spdk_accel_batch *_batch,
		void dst1, void dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb)
		{
		struct idxd_task idxd_task = (struct idxd_task )cb_arg;
		struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);
		struct idxd_batch batch = (struct idxd_batch )_batch;

		idxd_task->cb = cb;

		return spdk_idxd_batch_prep_dualcast(chan->chan, batch, dst1, dst2, src, nbytes, idxd_done,
		idxd_task);
		}

		static struct spdk_accel_engine idxd_accel_engine = {
		.get_capabilities = idxd_get_capabilities,
		.copy = idxd_submit_copy,
		.batch_get_max = idxd_batch_get_max,
		.batch_create = idxd_batch_start,
		.batch_prep_copy = idxd_batch_prep_copy,
		.batch_prep_dualcast = idxd_batch_prep_dualcast,
		.batch_submit = idxd_batch_submit,
		.dualcast = idxd_submit_dualcast,
		.compare = idxd_submit_compare,