Commit 9a562043 authored by Konrad Sztyber's avatar Konrad Sztyber Committed by Jim Harris
Browse files

accel: alloc buffers for data in remote memory domains 



If a task has buffers in a remote memory domains, we'll now allocate a
buffer from local memory and replace the original buffer with it.  This
is the first step in supporting buffers in remote memory domains. To
fully support it, we'll also need to pull/push the data before/after
executing a task.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I3c86bbb6dbe6a31cb2cae8ce7d73e272ddc2734c
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/15944


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 95707610

include/spdk_internal/accel_module.h

+17 −0
Original line number Diff line number Diff line
@@ -32,6 +32,19 @@ struct spdk_accel_crypto_key { 
	TAILQ_ENTRY(spdk_accel_crypto_key) link;

};



/**

 * Describes user's buffers in remote memory domains in case a module doesn't support memory domains

 * and accel needs to pull/push the data before submitting a task.  Should only be used by accel

 * itself and should not be touched by accel modules.

 */

struct spdk_accel_bounce_buffer {

	struct iovec			*orig_iovs;

	uint32_t			orig_iovcnt;

	struct spdk_memory_domain	*orig_domain;

	void				*orig_domain_ctx;

	struct iovec			iov;

};



enum spdk_accel_aux_iov_type {

	SPDK_ACCEL_AUX_IOV_SRC,

	SPDK_ACCEL_AUX_IOV_DST,
@@ -78,6 +91,10 @@ struct spdk_accel_task { 
		uint32_t		*output_size;

		uint32_t		block_size; /* for crypto op */

	};

	struct {

		struct spdk_accel_bounce_buffer s;

		struct spdk_accel_bounce_buffer d;

	} bounce;

	enum accel_opcode		op_code;

	uint64_t			iv; /* Initialization vector (tweak) for crypto op */

	int				flags;

lib/accel/accel.c

+134 −0
Original line number Diff line number Diff line
@@ -86,6 +86,7 @@ struct spdk_accel_sequence { 
	struct accel_io_channel			*ch;

	struct accel_sequence_tasks		tasks;

	struct accel_sequence_tasks		completed;

	TAILQ_HEAD(, accel_buffer)		bounce_bufs;

	spdk_accel_completion_cb		cb_fn;

	void					*cb_arg;

	TAILQ_ENTRY(spdk_accel_sequence)	link;
@@ -666,6 +667,7 @@ accel_sequence_get(struct accel_io_channel *ch) 


	TAILQ_INIT(&seq->tasks);

	TAILQ_INIT(&seq->completed);

	TAILQ_INIT(&seq->bounce_bufs);



	seq->ch = ch;
@@ -676,6 +678,13 @@ static inline void 
accel_sequence_put(struct spdk_accel_sequence *seq)

{

	struct accel_io_channel *ch = seq->ch;

	struct accel_buffer *buf;



	while (!TAILQ_EMPTY(&seq->bounce_bufs)) {

		buf = TAILQ_FIRST(&seq->bounce_bufs);

		TAILQ_REMOVE(&seq->bounce_bufs, buf, link);

		accel_put_buf(seq->ch, buf);

	}



	assert(TAILQ_EMPTY(&seq->tasks));

	assert(TAILQ_EMPTY(&seq->completed));
@@ -1041,6 +1050,119 @@ accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_ 
	return true;

}



static inline uint64_t

accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt)

{

	uint64_t result = 0;

	uint32_t i;



	for (i = 0; i < iovcnt; ++i) {

		result += iovs[i].iov_len;

	}



	return result;

}



static inline void

accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs,

			uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx,

			struct accel_buffer *buf)

{

	bounce->orig_iovs = *iovs;

	bounce->orig_iovcnt = *iovcnt;

	bounce->orig_domain = *domain;

	bounce->orig_domain_ctx = *domain_ctx;

	bounce->iov.iov_base = buf->buf;

	bounce->iov.iov_len = buf->len;



	*iovs = &bounce->iov;

	*iovcnt = 1;

	*domain = NULL;

}



static void

accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)

{

	struct spdk_accel_task *task;

	struct accel_buffer *accel_buf;



	accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);

	assert(accel_buf->buf == NULL);

	accel_buf->buf = buf;



	task = TAILQ_FIRST(&accel_buf->seq->tasks);

	assert(task != NULL);



	accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain,

				&task->src_domain_ctx, accel_buf);

	accel_process_sequence(accel_buf->seq);

}



static void

accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf)

{

	struct spdk_accel_task *task;

	struct accel_buffer *accel_buf;



	accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf);

	assert(accel_buf->buf == NULL);

	accel_buf->buf = buf;



	task = TAILQ_FIRST(&accel_buf->seq->tasks);

	assert(task != NULL);



	accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain,

				&task->dst_domain_ctx, accel_buf);

	accel_process_sequence(accel_buf->seq);

}



static int

accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task)

{

	struct accel_buffer *buf;



	if (task->src_domain != NULL) {

		/* By the time we're here, accel buffers should have been allocated */

		assert(task->src_domain != g_accel_domain);



		buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt));

		if (buf == NULL) {

			SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");

			return -ENOMEM;

		}



		TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link);

		if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) {

			return -EAGAIN;

		}



		accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt,

					&task->src_domain, &task->src_domain_ctx, buf);

	}



	if (task->dst_domain != NULL) {

		/* By the time we're here, accel buffers should have been allocated */

		assert(task->dst_domain != g_accel_domain);



		buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt));

		if (buf == NULL) {

			/* The src buffer will be released when a sequence is completed */

			SPDK_ERRLOG("Couldn't allocate buffer descriptor\n");

			return -ENOMEM;

		}



		TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link);

		if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) {

			return -EAGAIN;

		}



		accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt,

					&task->dst_domain, &task->dst_domain_ctx, buf);

	}



	return 0;

}



static void

accel_process_sequence(struct spdk_accel_sequence *seq)

{
@@ -1062,6 +1184,18 @@ accel_process_sequence(struct spdk_accel_sequence *seq) 
		return;

	}



	/* For now, assume that none of the modules support memory domains */

	rc = accel_sequence_check_bouncebuf(seq, task);

	if (rc != 0) {

		if (rc == -EAGAIN) {

			/* We couldn't allocate a buffer, wait until one is available */

			return;

		}



		accel_sequence_complete(seq, rc);

		return;

	}



	SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n",

		      g_opcode_strings[task->op_code], seq);