Commit 7afa4667 authored by Krzysztof Karas's avatar Krzysztof Karas Committed by Konrad Sztyber
Browse files

lib/accel: add spdk_accel_append_dix_generate/verify 



Add new APIs spdk_accel_append_dix_generate/verify to accel library.

Change-Id: Ic8f66a486383428eae8f4a846a95cac45e09b2e1
Signed-off-by: default avatarKrzysztof Karas <krzysztof.karas@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/24213


Community-CI: Mellanox Build Bot
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarSlawomir Ptak <slawomir.ptak@intel.com>
parent 80dfab0a

include/spdk/accel.h

+63 −0
Original line number Diff line number Diff line
@@ -944,6 +944,69 @@ int spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **seq, 
					uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

					spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append DIX generate operation to a sequence.

 *

 * \param seq Sequence object. If NULL, a new sequence object will be created.

 * \param ch I/O channel.

 * \param iovs Source I/O vector array. The total allocated memory size needs to be at least:

 *	num_blocks * block_size_no_md.

 * \param iovcnt Size of the source I/O vectors' array.

 * \param domain Memory domain to which the source buffers belong.

 * \param domain_ctx Source buffer domain context.

 * \param md_iov Metadata iovec. The total allocated memory size needs to be at least:

 *	num_blocks * md_size (8B or 16B, depending on the PI format).

 * \param md_domain Memory domain to which the metadata buffers belongs.

 * \param md_domain_ctx Metadata buffer domain context.

 * \param num_blocks Number of data blocks to process.

 * \param ctx DIX context. Contains the DIX configuration values, including the reference

 *	Application Tag value and initial value of the Reference Tag to insert.

 * \param cb_fn Called when this operation completes.

 * \param cb_arg Callback argument.

 *

 * \returns 0 on success, negative errno on failure.

 */

int spdk_accel_append_dix_generate(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

				   struct iovec *iovs, size_t iovcnt,

				   struct spdk_memory_domain *domain, void *domain_ctx,

				   struct iovec *md_iov, struct spdk_memory_domain *md_domain,

				   void *md_domain_ctx, uint32_t num_blocks,

				   const struct spdk_dif_ctx *ctx,

				   spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append DIX verify operation to a sequence.

 *

 * \param seq Sequence object. If NULL, a new sequence object will be created.

 * \param ch I/O channel.

 * \param iovs Source I/O vector array. The total allocated memory size needs to be at least:

 *	num_blocks * block_size_no_md.

 * \param iovcnt Size of the source I/O vectors' array.

 * \param domain Memory domain to which the source buffers belong.

 * \param domain_ctx Source buffer domain context.

 * \param md_iov Metadata iovec. The total allocated memory size needs to be at least:

 *	num_blocks * md_size (8B or 16B, depending on the PI format).

 * \param md_domain Memory domain to which the metadata buffers belongs.

 * \param md_domain_ctx Metadata buffer domain context.

 * \param num_blocks Number of data blocks to process.

 * \param ctx DIX context. Contains the DIX configuration values, including the reference

 *	Application Tag value and initial value of the Reference Tag to insert.

 * \param err DIX error detailed information.

 *	Note: the user must ensure the validity of this pointer throughout the entire

 *	operation because it is not validated along the processing path.

 * \param cb_fn Called when this operation completes.

 * \param cb_arg Callback argument.

 *

 * \returns 0 on success, negative errno on failure.

 */

int spdk_accel_append_dix_verify(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

				 struct iovec *iovs, size_t iovcnt,

				 struct spdk_memory_domain *domain, void *domain_ctx,

				 struct iovec *md_iov, struct spdk_memory_domain *md_domain,

				 void *md_domain_ctx, uint32_t num_blocks,

				 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err,

				 spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Finish a sequence and execute all its operations. After the completion callback is executed, the

 * sequence object is automatically freed.

lib/accel/accel.c

+103 −2
Original line number Diff line number Diff line
@@ -1666,6 +1666,103 @@ spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **pseq, struct sp 
	return 0;

}



int

spdk_accel_append_dix_generate(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

			       struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,

			       void *domain_ctx, struct iovec *md_iov,

			       struct spdk_memory_domain *md_domain, void *md_domain_ctx,

			       uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

			       spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *pseq = *seq;



	if (pseq == NULL) {

		pseq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(pseq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(pseq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*seq == NULL) {

			accel_sequence_put(pseq);

		}



		return -ENOMEM;

	}



	task->d.iovs = md_iov;

	task->d.iovcnt = 1;

	task->dst_domain = md_domain;

	task->dst_domain_ctx = md_domain_ctx;

	task->s.iovs = iovs;

	task->s.iovcnt = iovcnt;

	task->src_domain = domain;

	task->src_domain_ctx = domain_ctx;

	task->dif.ctx = ctx;

	task->dif.num_blocks = num_blocks;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE;



	TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);

	*seq = pseq;



	return 0;

}



int

spdk_accel_append_dix_verify(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

			     struct iovec *iovs, size_t iovcnt, struct spdk_memory_domain *domain,

			     void *domain_ctx, struct iovec *md_iov,

			     struct spdk_memory_domain *md_domain, void *md_domain_ctx,

			     uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

			     struct spdk_dif_error *err, spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *pseq = *seq;



	if (pseq == NULL) {

		pseq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(pseq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(pseq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, pseq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*seq == NULL) {

			accel_sequence_put(pseq);

		}



		return -ENOMEM;

	}



	task->d.iovs = md_iov;

	task->d.iovcnt = 1;

	task->dst_domain = md_domain;

	task->dst_domain_ctx = md_domain_ctx;

	task->s.iovs = iovs;

	task->s.iovcnt = iovcnt;

	task->src_domain = domain;

	task->src_domain_ctx = domain_ctx;

	task->dif.ctx = ctx;

	task->dif.err = err;

	task->dif.num_blocks = num_blocks;

	task->nbytes = num_blocks * ctx->block_size;

	task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY;



	TAILQ_INSERT_TAIL(&pseq->tasks, task, seq_link);

	*seq = pseq;



	return 0;

}



int

spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,

		   struct spdk_memory_domain **domain, void **domain_ctx)
@@ -2364,7 +2461,9 @@ accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next 
		task->dst_domain_ctx = next->dst_domain_ctx;

		break;

	case SPDK_ACCEL_OPC_CRC32C:

		/* crc32 is special, because it doesn't have a dst buffer */

	case SPDK_ACCEL_OPC_DIX_GENERATE:

	case SPDK_ACCEL_OPC_DIX_VERIFY:

		/* crc32 and dix_generate/verify are special, because they do not have a dst buffer */

		if (task->src_domain != next->src_domain) {

			return false;

		}
@@ -2372,7 +2471,7 @@ accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next 
					next->s.iovs, next->s.iovcnt)) {

			return false;

		}

		/* We can only change crc32's buffer if we can change previous task's buffer */

		/* We can only change operation's buffer if we can change previous task's buffer */

		prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);

		if (prev == NULL) {

			return false;
@@ -2434,6 +2533,8 @@ accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_ta 
	case SPDK_ACCEL_OPC_CRC32C:

	case SPDK_ACCEL_OPC_DIF_GENERATE_COPY:

	case SPDK_ACCEL_OPC_DIF_VERIFY_COPY:

	case SPDK_ACCEL_OPC_DIX_GENERATE:

	case SPDK_ACCEL_OPC_DIX_VERIFY:

		/* We can only merge tasks when one of them is a copy */

		if (next->op_code != SPDK_ACCEL_OPC_COPY) {

			break;

lib/accel/spdk_accel.map

+2 −0
Original line number Diff line number Diff line
@@ -37,6 +37,8 @@ 
	spdk_accel_append_dif_verify_copy;

	spdk_accel_append_dif_generate;

	spdk_accel_append_dif_generate_copy;

	spdk_accel_append_dix_generate;

	spdk_accel_append_dix_verify;

	spdk_accel_sequence_finish;

	spdk_accel_sequence_abort;

	spdk_accel_sequence_reverse;

test/unit/lib/accel/accel.c/accel_ut.c

+216 −0
Original line number Diff line number Diff line
@@ -4118,6 +4118,220 @@ test_sequence_crc32(void) 
	poll_threads();

}



static void

test_sequence_dix_generate_verify(void)

{

	struct spdk_accel_sequence *seq;

	struct spdk_io_channel *ioch;

	struct ut_sequence ut_seq = {};

	char srcbuf[3][4096], dstbuf[3][4096];

	struct iovec src_iovs[3], dst_iovs[3];

	uint32_t block_size = 512, md_size = 8;

	struct spdk_dif_ctx_init_ext_opts dif_opts;

	struct spdk_dif_ctx dif_ctx = {};

	struct spdk_dif_error dif_err;

	int i, rc, completed = 0;



	ioch = spdk_accel_get_io_channel();

	SPDK_CU_ASSERT_FATAL(ioch != NULL);



	for (i = 0; i < 3; i++) {

		memset(srcbuf[i], 0xdead, sizeof(srcbuf[i]));

		memset(dstbuf[i], 0, sizeof(dstbuf[i]));

		src_iovs[i].iov_base = srcbuf[i];

		src_iovs[i].iov_len = sizeof(srcbuf[i]);

		dst_iovs[i].iov_base = dstbuf[i];

		dst_iovs[i].iov_len = sizeof(dstbuf[i]);

	}



	dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);

	dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;



	rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1,

			       SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK |

			       SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts);

	SPDK_CU_ASSERT_FATAL(rc == 0);



	seq = NULL;

	completed = 0;

	for (i = 0; i < 3; i++) {

		rc = spdk_accel_append_dix_generate(&seq, ioch, &dst_iovs[i], 1, NULL, NULL,

						    &src_iovs[i], NULL, NULL,

						    src_iovs[i].iov_len / block_size, &dif_ctx,

						    ut_sequence_step_cb, &completed);

		CU_ASSERT_EQUAL(rc, 0);

		rc = spdk_accel_append_dix_verify(&seq, ioch, &dst_iovs[i], 1, NULL, NULL,

						  &src_iovs[i], NULL, NULL,

						  src_iovs[i].iov_len / block_size, &dif_ctx,

						  &dif_err, ut_sequence_step_cb, &completed);

		CU_ASSERT_EQUAL(rc, 0);

	}



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 6);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);



	/* DIX metadata should be equal for the same source buffers and tags */

	CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[1], 4096), 0);

	CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[2], 4096), 0);



	ut_clear_operations();

	spdk_put_io_channel(ioch);

	poll_threads();

}



static void

test_sequence_dix(void)

{

	struct spdk_accel_sequence *seq = NULL;

	struct spdk_io_channel *ioch;

	struct ut_sequence ut_seq = {};

	char buf[3][4096], md_buf[64];

	struct iovec iovs[3], md_iov;

	uint32_t block_size = 512, md_size = 8;

	struct spdk_dif_ctx_init_ext_opts dif_opts;

	struct spdk_dif_ctx dif_ctx = {};

	struct spdk_dif_error dif_err;

	struct spdk_accel_crypto_key *key;

	struct spdk_accel_crypto_key_create_param key_params = {

		.cipher = "AES_XTS",

		.hex_key = "00112233445566778899aabbccddeeff",

		.hex_key2 = "ffeeddccbbaa99887766554433221100",

		.key_name = "ut_key",

	};

	int i, rc, completed = 0;

	struct accel_module modules[SPDK_ACCEL_OPC_LAST];



	ioch = spdk_accel_get_io_channel();

	SPDK_CU_ASSERT_FATAL(ioch != NULL);



	rc = spdk_accel_crypto_key_create(&key_params);

	CU_ASSERT_EQUAL(rc, 0);

	key = spdk_accel_crypto_key_get(key_params.key_name);

	SPDK_CU_ASSERT_FATAL(key != NULL);



	/* Override the submit_tasks function. */

	g_module_if.submit_tasks = ut_sequence_submit_tasks;

	for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {

		g_seq_operations[i].complete_status = 0;

		g_seq_operations[i].submit_status = 0;

		g_seq_operations[i].count = 0;



		modules[i] = g_modules_opc[i];

		g_modules_opc[i] = g_module;

	}



	for (i = 0; i < 3; i++) {

		memset(buf[i], 0, sizeof(buf[i]));

		iovs[i].iov_base = buf[i];

		iovs[i].iov_len = sizeof(buf[i]);

	}

	memset(md_buf, 0, sizeof(md_buf));

	md_iov.iov_base = md_buf;

	md_iov.iov_len = sizeof(md_buf);



	/* Prepare first source buffer. */

	memset(iovs[0].iov_base, 0xde, iovs[0].iov_len);



	g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0;

	g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0;

	g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1;

	g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1;

	g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &iovs[0];

	/* Copy will be skipped, so the destination buffer of encrypt will change. */

	g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &iovs[2];



	rc = spdk_accel_append_encrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL,

				       &iovs[0], 1, NULL, NULL, 0, block_size,

				       ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);

	dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16;



	rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1,

			       SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK |

			       SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts);

	SPDK_CU_ASSERT_FATAL(rc == 0);



	rc = spdk_accel_append_dix_generate(&seq, ioch, &iovs[1], 1, NULL, NULL,

					    &md_iov, NULL, NULL, iovs[1].iov_len / block_size,

					    &dif_ctx, ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL,

				    &iovs[1], 1, NULL, NULL,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();



	CU_ASSERT_EQUAL(completed, 3);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0);



	seq = NULL;

	completed = 0;



	/* This time we start with first iovec containing encrypted data from previous sequence. */

	memcpy(iovs[0].iov_base, iovs[2].iov_base, iovs[0].iov_len);



	g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0;

	g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0;

	g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1;

	g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1;

	g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &iovs[0];

	/* Copy will be skipped, so the destination buffer of decrypt will change. */

	g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &iovs[2];



	rc = spdk_accel_append_decrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL,

				       &iovs[0], 1, NULL, NULL, 0, block_size,

				       ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	rc = spdk_accel_append_dix_verify(&seq, ioch, &iovs[1], 1, NULL, NULL, &md_iov, NULL, NULL,

					  iovs[1].iov_len / block_size, &dif_ctx, &dif_err,

					  ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL, &iovs[1], 1, NULL, NULL,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();



	CU_ASSERT_EQUAL(completed, 3);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0);



	ut_clear_operations();



	/* Cleanup module pointers to make subsequent tests work correctly. */

	for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) {

		g_modules_opc[i] = modules[i];

	}



	rc = spdk_accel_crypto_key_destroy(key);

	CU_ASSERT_EQUAL(rc, 0);



	spdk_put_io_channel(ioch);

	poll_threads();

}



static int

test_sequence_setup(void)

{
@@ -4208,6 +4422,8 @@ main(int argc, char **argv) 
	CU_ADD_TEST(seq_suite, test_sequence_driver);

	CU_ADD_TEST(seq_suite, test_sequence_same_iovs);

	CU_ADD_TEST(seq_suite, test_sequence_crc32);

	CU_ADD_TEST(seq_suite, test_sequence_dix_generate_verify);

	CU_ADD_TEST(seq_suite, test_sequence_dix);



	suite = CU_add_suite("accel", test_setup, test_cleanup);

	CU_ADD_TEST(suite, test_spdk_accel_task_complete);