Commit fe705480 authored by Ben Walker's avatar Ben Walker Committed by Tomasz Zawadzki
Browse files

idxd: Add support for vectored copy operations 



Change-Id: Icb650129488b3cea76cf9082c02667f5b13b5ab4
Signed-off-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10286


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
parent e1c20e37

CHANGELOG.md

+4 −0
Original line number Diff line number Diff line
@@ -7,6 +7,10 @@ 
The batching capability was removed. Batching is now considered an implementation

detail of the low level drivers.



### idxd



Many APIs are now vectored rather than scalar, meaning they take iovecs instead of individual pointers.



### nvme



API `spdk_nvme_trtype_is_fabrics` was added to return existing transport type

examples/idxd/perf/perf.c

+8 −2
Original line number Diff line number Diff line
@@ -609,6 +609,8 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) 
{

	int random_num;

	int rc = 0;

	struct iovec siov = {};

	struct iovec diov = {};



	assert(t);

	t->current_queue_depth++;
@@ -621,8 +623,12 @@ _submit_ops(struct idxd_chan_entry *t, struct idxd_task *task) 


		switch (g_workload_selection) {

		case IDXD_COPY:

			rc = spdk_idxd_submit_copy(t->ch, task->dst, task->src,

						   g_xfer_size_bytes, idxd_done, task);

			siov.iov_base = task->src;

			siov.iov_len = g_xfer_size_bytes;

			diov.iov_base = task->dst;

			diov.iov_len = g_xfer_size_bytes;

			rc = spdk_idxd_submit_copy(t->ch, &diov, 1, &siov, 1,

						   idxd_done, task);

			break;

		case IDXD_FILL:

			/* For fill use the first byte of the task->dst buffer */

include/spdk/idxd.h

+6 −4
Original line number Diff line number Diff line
@@ -217,9 +217,10 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd 
 * by writing to the proper device portal.

 *

 * \param chan IDXD channel to submit request.

 * \param dst Destination virtual address.

 * \param src Source virtual address.

 * \param nbytes Number of bytes to copy.

 * \param diov Destination iovec

 * \param diovcnt Number of elements in diov

 * \param siov Source iovec

 * \param siovcnt Number of elements in siov

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the arg parameter in

 * the completion callback.
@@ -227,7 +228,8 @@ int spdk_idxd_batch_prep_dualcast(struct spdk_idxd_io_channel *chan, struct idxd 
 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,

			  void *dst, const void *src, uint64_t nbytes,

			  struct iovec *diov, uint32_t diovcnt,

			  struct iovec *siov, uint32_t siovcnt,

			  spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

lib/idxd/Makefile

+1 −1
Original line number Diff line number Diff line
@@ -34,7 +34,7 @@ 
SPDK_ROOT_DIR := $(abspath $(CURDIR)/../..)

include $(SPDK_ROOT_DIR)/mk/spdk.common.mk



SO_VER := 6

SO_VER := 7

SO_MINOR := 0



C_SRCS = idxd.c idxd_user.c

lib/idxd/idxd.c

+69 −4
Original line number Diff line number Diff line
@@ -456,9 +456,8 @@ _idxd_batch_prep_nop(struct spdk_idxd_io_channel *chan, struct idxd_batch *batch 
	return 0;

}





int

spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void *src,

static inline int

_idxd_submit_copy_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,

			 uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;
@@ -502,6 +501,72 @@ error: 
	return rc;

}



int

spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan,

		      struct iovec *diov, uint32_t diovcnt,

		      struct iovec *siov, uint32_t siovcnt,

		      spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_ops *op;

	void *src, *dst;

	uint64_t src_addr, dst_addr;

	int rc;

	uint64_t len;

	struct idxd_batch *batch;

	struct spdk_ioviter iter;



	assert(chan != NULL);

	assert(diov != NULL);

	assert(siov != NULL);



	if (diovcnt == 1 && siovcnt == 1) {

		/* Simple case - copying one buffer to another */

		if (diov[0].iov_len < siov[0].iov_len) {

			return -EINVAL;

		}



		return _idxd_submit_copy_single(chan, diov[0].iov_base,

						siov[0].iov_base, siov[0].iov_len,

						cb_fn, cb_arg);

	}



	batch = spdk_idxd_batch_create(chan);

	if (!batch) {

		return -EBUSY;

	}



	for (len = spdk_ioviter_first(&iter, siov, siovcnt, diov, diovcnt, &src, &dst);

	     len > 0;

	     len = spdk_ioviter_next(&iter, &src, &dst)) {

		rc = _idxd_prep_batch_cmd(chan, NULL, NULL, batch, &desc, &op);

		if (rc) {

			goto err;

		}



		rc = _vtophys(src, &src_addr, len);

		if (rc) {

			goto err;

		}



		rc = _vtophys(dst, &dst_addr, len);

		if (rc) {

			goto err;

		}



		desc->opcode = IDXD_OPCODE_MEMMOVE;

		desc->src_addr = src_addr;

		desc->dst_addr = dst_addr;

		desc->xfer_size = len;

	}



	return spdk_idxd_batch_submit(chan, batch, cb_fn, cb_arg);



err:

	spdk_idxd_batch_cancel(chan, batch);

	return rc;

}



/* Dual-cast copies the same source to two separate destination buffers. */

int

spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *dst2,