Commit f71d1c11 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Konrad Sztyber
Browse files

accel: Add append version for T10 DIF APIs 



Add append versions of dif_verify, dif_verify_copy, dif_generate, and
dif_generate_copy to public APIs.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I98a117577e2b9445390299b6ad1e6e8423efbf18
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23739


Reviewed-by: default avatarBen Walker <ben@nvidia.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
parent 5e4d957e

include/spdk/accel.h

+134 −0
Original line number Diff line number Diff line
@@ -663,6 +663,140 @@ int spdk_accel_append_crc32c(struct spdk_accel_sequence **seq, struct spdk_io_ch 
			     struct spdk_memory_domain *domain, void *domain_ctx,

			     uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append a Data Integrity Field (DIF) verify operation to a sequence.

 *

 * This operation computes the DIF on the data and compares it against the DIF contained

 * in the metadata.

 *

 * \param seq Sequence object.  If NULL, a new sequence object will be created.

 * \param ch I/O channel.

 * \param iovs The io vector array. The total allocated memory size needs to be at least:

 *             num_blocks * block_size (including metadata)

 * \param iovcnt The size of the io vectors array.

 * \param domain Memory domain to which the data buffers belong.

 * \param domain_ctx Data buffer domain context.

 * \param num_blocks Number of data blocks to check.

 * \param ctx DIF context. Contains the DIF configuration values, including the reference

 *            Application Tag value and initial value of the Reference Tag to check

 *            Note: the user must ensure the validity of this pointer throughout the entire operation

 *            because it is not validated along the processing path.

 * \param err DIF error detailed information.

 *            Note: the user must ensure the validity of this pointer throughout the entire operation

 *            because it is not validated along the processing path.

 * \param cb_fn Callback to be executed once this operation is completed.

 * \param cb_arg Argument to be passed to `cb_fn`.

 *

 * \return 0 if operation was successfully added to the sequence, negative errno on failure.

 */

int spdk_accel_append_dif_verify(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

				 struct iovec *iovs, size_t iovcnt,

				 struct spdk_memory_domain *domain, void *domain_ctx,

				 uint32_t num_blocks,

				 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,

				 spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append a Data Integrity Field (DIF) copy and verify operation to a sequence.

 *

 * This operation copies memory from the source to the destination address and removes

 * the DIF data with its verification according to the flags provided in the context.

 *

 * \param seq Sequence object.  If NULL, a new sequence object will be created.

 * \param ch I/O channel.

 * \param dst_iovs The destination I/O vector array. The total allocated memory size needs

 *                to be at least: num_blocks * block_size (provided to spdk_dif_ctx_init())

 * \param dst_iovcnt The size of the destination I/O vectors array.

 * \param dst_domain Memory domain to which the destination buffers belong.

 * \param dst_domain_ctx Destination buffer domain context.

 * \param src_iovs The source I/O vector array. The total allocated memory size needs

 *                to be at least: num_blocks * data_block_size.

 * \param src_iovcnt The size of the source I/O vectors array.

 * \param src_domain Memory domain to which the source buffers belong.

 * \param src_domain_ctx Source buffer domain context.

 * \param num_blocks Number of data blocks to process.

 * \param ctx DIF context. Contains the DIF configuration values, including the reference

 *            Application Tag value and initial value of the Reference Tag to insert.

 * \param err DIF error detailed information.

 *            Note: the user must ensure the validity of this pointer throughout the entire operation

 *            because it is not validated along the processing path.

 * \param cb_fn Called when this operation completes.

 * \param cb_arg Callback argument.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_append_dif_verify_copy(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

				      struct iovec *dst_iovs, size_t dst_iovcnt,

				      struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,

				      struct iovec *src_iovs, size_t src_iovcnt,

				      struct spdk_memory_domain *src_domain, void *src_domain_ctx,

				      uint32_t num_blocks,

				      const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,

				      spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append a Data Integrity Field (DIF) generate operation to a sequence.

 *

 * This operation compute the DIF on the source data and inserting the DIF in place into

 * the source data.

 *

 * \param seq Sequence object.  If NULL, a new sequence object will be created.

 * \param ch I/O channel associated with this call.

 * \param iovs The io vector array. The total allocated memory size needs to be at least:

 *             num_blocks * block_size (including metadata)

 * \param iovcnt The size of the io vectors array.

 * \param domain Memory domain to which the data buffers belong.

 * \param domain_ctx Data buffer domain context.

 * \param num_blocks Number of data blocks.

 * \param ctx DIF context. Contains the DIF configuration values, including the reference

 *            Application Tag value and initial value of the Reference Tag to insert

 * \param cb_fn Called when this operation completes.

 * \param cb_arg Callback argument.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_append_dif_generate(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

				   struct iovec *iovs, size_t iovcnt,

				   struct spdk_memory_domain *domain, void *domain_ctx,

				   uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

				   spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Submit a Data Integrity Field (DIF) copy and generate request.

 *

 * This operation copies memory from the source to the destination address,

 * while computing the DIF on the source data and inserting the DIF into

 * the output data.

 *

 * \param seq Sequence object.  If NULL, a new sequence object will be created.

 * \param ch I/O channel associated with this call.

 * \param dst_iovs The destination io vector array. The total allocated memory size needs

 *                to be at least: num_blocks * block_size (provided to spdk_dif_ctx_init())

 * \param dst_iovcnt The size of the destination io vectors array.

 * \param dst_domain Memory domain to which the destination buffers belong.

 * \param dst_domain_ctx Destination buffer domain context.

 * \param src_iovs The source io vector array. The total allocated memory size needs

 *                to be at least: num_blocks * data_block_size.

 * \param src_iovcnt The size of the source io vectors array.

 * \param src_domain Memory domain to which the source buffers belong.

 * \param src_domain_ctx Source buffer domain context.

 * \param num_blocks Number of data blocks to process.

 * \param ctx DIF context. Contains the DIF configuration values, including the reference

 *            Application Tag value and initial value of the Reference Tag to insert

 * \param cb_fn Called when this operation completes.

 * \param cb_arg Callback argument.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **seq,

					struct spdk_io_channel *ch,

					struct iovec *dst_iovs, size_t dst_iovcnt,

					struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,

					struct iovec *src_iovs, size_t src_iovcnt,

					struct spdk_memory_domain *src_domain, void *src_domain_ctx,

					uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

					spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Finish a sequence and execute all its operations. After the completion callback is executed, the

 * sequence object is automatically freed.

lib/accel/accel.c

+195 −1
Original line number Diff line number Diff line
@@ -1336,6 +1336,194 @@ spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_chann 
	return 0;

}



int

spdk_accel_append_dif_verify(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,

			     struct iovec *iovs, size_t iovcnt,

			     struct spdk_memory_domain *domain, void *domain_ctx,

			     uint32_t num_blocks,

			     const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,

			     spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *seq = *pseq;



	if (seq == NULL) {

		seq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(seq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(seq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*pseq == NULL) {

			accel_sequence_put(seq);

		}



		return -ENOMEM;

	}



	task->s.iovs = iovs;

	task->s.iovcnt = iovcnt;

	task->src_domain = domain;

	task->src_domain_ctx = domain_ctx;

	task->dst_domain = NULL;

	task->dif.ctx = ctx;

	task->dif.err = err;

	task->dif.num_blocks = num_blocks;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY;



	TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);

	*pseq = seq;



	return 0;

}



int

spdk_accel_append_dif_verify_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,

				  struct iovec *dst_iovs, size_t dst_iovcnt,

				  struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,

				  struct iovec *src_iovs, size_t src_iovcnt,

				  struct spdk_memory_domain *src_domain, void *src_domain_ctx,

				  uint32_t num_blocks,

				  const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,

				  spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *seq = *pseq;



	if (seq == NULL) {

		seq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(seq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(seq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*pseq == NULL) {

			accel_sequence_put(seq);

		}



		return -ENOMEM;

	}



	task->dst_domain = dst_domain;

	task->dst_domain_ctx = dst_domain_ctx;

	task->d.iovs = dst_iovs;

	task->d.iovcnt = dst_iovcnt;

	task->src_domain = src_domain;

	task->src_domain_ctx = src_domain_ctx;

	task->s.iovs = src_iovs;

	task->s.iovcnt = src_iovcnt;

	task->dif.ctx = ctx;

	task->dif.err = err;

	task->dif.num_blocks = num_blocks;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY;



	TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);

	*pseq = seq;



	return 0;

}



int

spdk_accel_append_dif_generate(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,

			       struct iovec *iovs, size_t iovcnt,

			       struct spdk_memory_domain *domain, void *domain_ctx,

			       uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

			       spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *seq = *pseq;



	if (seq == NULL) {

		seq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(seq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(seq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*pseq == NULL) {

			accel_sequence_put(seq);

		}



		return -ENOMEM;

	}



	task->s.iovs = iovs;

	task->s.iovcnt = iovcnt;

	task->src_domain = domain;

	task->src_domain_ctx = domain_ctx;

	task->dst_domain = NULL;

	task->dif.ctx = ctx;

	task->dif.num_blocks = num_blocks;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE;



	TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);

	*pseq = seq;



	return 0;

}



int

spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,

				    struct iovec *dst_iovs, size_t dst_iovcnt,

				    struct spdk_memory_domain *dst_domain, void *dst_domain_ctx,

				    struct iovec *src_iovs, size_t src_iovcnt,

				    struct spdk_memory_domain *src_domain, void *src_domain_ctx,

				    uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

				    spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *seq = *pseq;



	if (seq == NULL) {

		seq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(seq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(seq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*pseq == NULL) {

			accel_sequence_put(seq);

		}



		return -ENOMEM;

	}



	task->dst_domain = dst_domain;

	task->dst_domain_ctx = dst_domain_ctx;

	task->d.iovs = dst_iovs;

	task->d.iovcnt = dst_iovcnt;

	task->src_domain = src_domain;

	task->src_domain_ctx = src_domain_ctx;

	task->s.iovs = src_iovs;

	task->s.iovcnt = src_iovcnt;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY;



	TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);

	*pseq = seq;



	return 0;

}



int

spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,

		   struct spdk_memory_domain **domain, void **domain_ctx)
@@ -2019,6 +2207,8 @@ accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next 
	case SPDK_ACCEL_OPC_FILL:

	case SPDK_ACCEL_OPC_ENCRYPT:

	case SPDK_ACCEL_OPC_DECRYPT:

	case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:

	case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:

		if (task->dst_domain != next->src_domain) {

			return false;

		}
@@ -2077,7 +2267,9 @@ accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_ta 
		    next->op_code != SPDK_ACCEL_OPC_COPY &&

		    next->op_code != SPDK_ACCEL_OPC_ENCRYPT &&

		    next->op_code != SPDK_ACCEL_OPC_DECRYPT &&

		    next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C) {

		    next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C &&

		    next->op_code != SPDK_ACCEL_OPC_DIF_GENERATE_COPY &&

		    next->op_code != SPDK_ACCEL_OPC_DIF_VERIFY_COPY) {

			break;

		}

		if (task->dst_domain != next->src_domain) {
@@ -2098,6 +2290,8 @@ accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_ta 
	case SPDK_ACCEL_OPC_ENCRYPT:

	case SPDK_ACCEL_OPC_DECRYPT:

	case SPDK_ACCEL_OPC_CRC32C:

	case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:

	case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:

		/* We can only merge tasks when one of them is a copy */

		if (next->op_code != SPDK_ACCEL_OPC_COPY) {

			break;

lib/accel/spdk_accel.map

+4 −0
Original line number Diff line number Diff line
@@ -31,6 +31,10 @@ 
	spdk_accel_append_encrypt;

	spdk_accel_append_decrypt;

	spdk_accel_append_crc32c;

	spdk_accel_append_dif_verify;

	spdk_accel_append_dif_verify_copy;

	spdk_accel_append_dif_generate;

	spdk_accel_append_dif_generate_copy;

	spdk_accel_sequence_finish;

	spdk_accel_sequence_abort;

	spdk_accel_sequence_reverse;