Commit 9a377ecb authored by Konrad Sztyber's avatar Konrad Sztyber
Browse files

accel: append support for crc32c 



It is now possible to append an operation calculating crc32c to an accel
sequence.  A crc32c operation needs special care when it's part of a
sequence, because it doesn't have a destination buffer.  It means that
we can remove copy operations following crc32c only when it's possible
to change the dst buffer of the operation preceding crc32c.

Signed-off-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Change-Id: I29204ce52d635162d2202136609f8f8f33db312d
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/17427


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarChangpeng Liu <changpeng.liu@intel.com>
Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
parent 2b1ad70c

include/spdk/accel.h

+21 −0
Original line number Diff line number Diff line
@@ -471,6 +471,27 @@ int spdk_accel_append_decrypt(struct spdk_accel_sequence **seq, struct spdk_io_c 
			      uint64_t iv, uint32_t block_size, int flags,

			      spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Append a crc32c operation to a sequence.

 *

 * \param seq Sequence object.  If NULL, a new sequence object will be created.

 * \param ch I/O channel.

 * \param dst Destination to write the calculated value.

 * \param iovs Source I/O vector array.

 * \param iovcnt Size of the `iovs` array.

 * \param domain Memory domain to which the source buffers belong.

 * \param domain_ctx Source buffer domain context.

 * \param seed Initial value.

 * \param cb_fn Callback to be executed once this operation is completed.

 * \param cb_arg Argument to be passed to `cb_fn`.

 *

 * \return 0 if operation was successfully added to the sequence, negative errno otherwise.

 */

int spdk_accel_append_crc32c(struct spdk_accel_sequence **seq, struct spdk_io_channel *ch,

			     uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,

			     struct spdk_memory_domain *domain, void *domain_ctx,

			     uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg);



/**

 * Finish a sequence and execute all its operations. After the completion callback is executed, the

 * sequence object is automatically freed.

lib/accel/accel.c

+69 −1
Original line number Diff line number Diff line
@@ -1085,6 +1085,48 @@ spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_chan 
	return 0;

}



int

spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch,

			 uint32_t *dst, struct iovec *iovs, uint32_t iovcnt,

			 struct spdk_memory_domain *domain, void *domain_ctx,

			 uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg)

{

	struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch);

	struct spdk_accel_task *task;

	struct spdk_accel_sequence *seq = *pseq;



	if (seq == NULL) {

		seq = accel_sequence_get(accel_ch);

		if (spdk_unlikely(seq == NULL)) {

			return -ENOMEM;

		}

	}



	assert(seq->ch == accel_ch);

	task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg);

	if (spdk_unlikely(task == NULL)) {

		if (*pseq == NULL) {

			accel_sequence_put(seq);

		}



		return -ENOMEM;

	}



	task->s.iovs = iovs;

	task->s.iovcnt = iovcnt;

	task->src_domain = domain;

	task->src_domain_ctx = domain_ctx;

	task->crc_dst = dst;

	task->seed = seed;

	task->op_code = ACCEL_OPC_CRC32C;

	task->dst_domain = NULL;



	TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link);

	*pseq = seq;



	return 0;

}



int

spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf,

		   struct spdk_memory_domain **domain, void **domain_ctx)
@@ -1726,6 +1768,8 @@ accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uin 
static bool

accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next)

{

	struct spdk_accel_task *prev;



	switch (task->op_code) {

	case ACCEL_OPC_DECOMPRESS:

	case ACCEL_OPC_FILL:
@@ -1743,6 +1787,28 @@ accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next 
		task->dst_domain = next->dst_domain;

		task->dst_domain_ctx = next->dst_domain_ctx;

		break;

	case ACCEL_OPC_CRC32C:

		/* crc32 is special, because it doesn't have a dst buffer */

		if (task->src_domain != next->src_domain) {

			return false;

		}

		if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt,

					next->s.iovs, next->s.iovcnt)) {

			return false;

		}

		/* We can only change crc32's buffer if we can change previous task's buffer */

		prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link);

		if (prev == NULL) {

			return false;

		}

		if (!accel_task_set_dstbuf(prev, next)) {

			return false;

		}

		task->s.iovs = next->d.iovs;

		task->s.iovcnt = next->d.iovcnt;

		task->src_domain = next->dst_domain;

		task->src_domain_ctx = next->dst_domain_ctx;

		break;

	default:

		return false;

	}
@@ -1766,7 +1832,8 @@ accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_ta 
		if (next->op_code != ACCEL_OPC_DECOMPRESS &&

		    next->op_code != ACCEL_OPC_COPY &&

		    next->op_code != ACCEL_OPC_ENCRYPT &&

		    next->op_code != ACCEL_OPC_DECRYPT) {

		    next->op_code != ACCEL_OPC_DECRYPT &&

		    next->op_code != ACCEL_OPC_CRC32C) {

			break;

		}

		if (task->dst_domain != next->src_domain) {
@@ -1787,6 +1854,7 @@ accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_ta 
	case ACCEL_OPC_FILL:

	case ACCEL_OPC_ENCRYPT:

	case ACCEL_OPC_DECRYPT:

	case ACCEL_OPC_CRC32C:

		/* We can only merge tasks when one of them is a copy */

		if (next->op_code != ACCEL_OPC_COPY) {

			break;

lib/accel/spdk_accel.map

+1 −0
Original line number Diff line number Diff line
@@ -26,6 +26,7 @@ 
	spdk_accel_append_decompress;

	spdk_accel_append_encrypt;

	spdk_accel_append_decrypt;

	spdk_accel_append_crc32c;

	spdk_accel_sequence_finish;

	spdk_accel_sequence_abort;

	spdk_accel_sequence_reverse;

test/unit/lib/accel/accel.c/accel_ut.c

+233 −0
Original line number Diff line number Diff line
@@ -3768,6 +3768,238 @@ test_sequence_same_iovs(void) 
	poll_threads();

}



static void

test_sequence_crc32(void)

{

	struct spdk_accel_sequence *seq = NULL;

	struct spdk_io_channel *ioch;

	struct ut_sequence ut_seq;

	struct accel_module modules[ACCEL_OPC_LAST];

	char buf[4096], tmp[3][4096];

	struct iovec src_iovs[4], dst_iovs[4];

	uint32_t crc, crc2;

	int i, rc, completed;



	ioch = spdk_accel_get_io_channel();

	SPDK_CU_ASSERT_FATAL(ioch != NULL);



	/* Override the submit_tasks function */

	g_module_if.submit_tasks = ut_sequnce_submit_tasks;

	for (i = 0; i < ACCEL_OPC_LAST; ++i) {

		g_seq_operations[i].submit = sw_accel_submit_tasks;

		modules[i] = g_modules_opc[i];

		g_modules_opc[i] = g_module;

	}

	g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress;



	/* First check the simplest case - single crc32c operation */

	seq = NULL;

	completed = 0;

	crc = 0;

	memset(buf, 0xa5, sizeof(buf));



	src_iovs[0].iov_base = buf;

	src_iovs[0].iov_len = sizeof(buf);

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 1);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1);

	CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u));

	g_seq_operations[ACCEL_OPC_CRC32C].count = 0;



	/* Now check copy+crc - this should remove the copy operation and change the buffer for the

	 * crc operation */

	seq = NULL;

	completed = 0;

	crc = 0;

	memset(buf, 0x5a, sizeof(buf));

	memset(&tmp[0], 0, sizeof(tmp[0]));



	dst_iovs[0].iov_base = tmp[0];

	dst_iovs[0].iov_len = sizeof(tmp[0]);

	src_iovs[0].iov_base = buf;

	src_iovs[0].iov_len = sizeof(buf);

	rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL,

				    &src_iovs[0], 1, NULL, NULL, 0,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	src_iovs[1].iov_base = tmp[0];

	src_iovs[1].iov_len = sizeof(tmp[0]);

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 2);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0);

	CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u));

	g_seq_operations[ACCEL_OPC_CRC32C].count = 0;



	/* Check crc+copy - this time the copy cannot be removed, because there's no operation

	 * before crc to change the buffer */

	seq = NULL;

	completed = 0;

	crc = 0;

	memset(buf, 0, sizeof(buf));

	memset(&tmp[0], 0xa5, sizeof(tmp[0]));



	src_iovs[0].iov_base = tmp[0];

	src_iovs[0].iov_len = sizeof(tmp[0]);

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	dst_iovs[1].iov_base = buf;

	dst_iovs[1].iov_len = sizeof(buf);

	src_iovs[1].iov_base = tmp[0];

	src_iovs[1].iov_len = sizeof(tmp[0]);

	rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL,

				    &src_iovs[1], 1, NULL, NULL, 0,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 2);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1);

	CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u));

	CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0);

	g_seq_operations[ACCEL_OPC_CRC32C].count = 0;

	g_seq_operations[ACCEL_OPC_COPY].count = 0;



	/* Check a sequence with an operation at the beginning that can have its buffer changed, two

	 * crc operations and a copy at the end.  The copy should be removed and the dst buffer of

	 * the first operation and the src buffer of the crc operations should be changed.

	 */

	seq = NULL;

	completed = 0;

	crc = crc2 = 0;

	memset(buf, 0, sizeof(buf));

	memset(&tmp[0], 0x5a, sizeof(tmp[0]));

	dst_iovs[0].iov_base = tmp[1];

	dst_iovs[0].iov_len = sizeof(tmp[1]);

	src_iovs[0].iov_base = tmp[0];

	src_iovs[0].iov_len = sizeof(tmp[0]);

	rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL,

					  &src_iovs[0], 1, NULL, NULL, 0,

					  ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	src_iovs[1].iov_base = tmp[1];

	src_iovs[1].iov_len = sizeof(tmp[1]);

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	src_iovs[2].iov_base = tmp[1];

	src_iovs[2].iov_len = sizeof(tmp[1]);

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	dst_iovs[3].iov_base = buf;

	dst_iovs[3].iov_len = sizeof(buf);

	src_iovs[3].iov_base = tmp[1];

	src_iovs[3].iov_len = sizeof(tmp[1]);

	rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL,

				    &src_iovs[3], 1, NULL, NULL, 0,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 4);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 2);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0);

	CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u));

	CU_ASSERT_EQUAL(crc, crc2);

	CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0);

	g_seq_operations[ACCEL_OPC_CRC32C].count = 0;

	g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0;



	/* Check that a copy won't be removed if the buffers don't match */

	seq = NULL;

	completed = 0;

	crc = 0;

	memset(buf, 0, sizeof(buf));

	memset(&tmp[0], 0xa5, 2048);

	memset(&tmp[1], 0xfe, sizeof(tmp[1]));

	memset(&tmp[2], 0xfe, sizeof(tmp[1]));

	dst_iovs[0].iov_base = &tmp[1][2048];

	dst_iovs[0].iov_len = 2048;

	src_iovs[0].iov_base = tmp[0];

	src_iovs[0].iov_len = 2048;

	rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL,

					  &src_iovs[0], 1, NULL, NULL, 0,

					  ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	src_iovs[1].iov_base = &tmp[1][2048];

	src_iovs[1].iov_len = 2048;

	rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0,

				      ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	dst_iovs[2].iov_base = buf;

	dst_iovs[2].iov_len = sizeof(buf);

	src_iovs[2].iov_base = tmp[1];

	src_iovs[2].iov_len = sizeof(tmp[1]);

	rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL,

				    &src_iovs[2], 1, NULL, NULL, 0,

				    ut_sequence_step_cb, &completed);

	CU_ASSERT_EQUAL(rc, 0);



	ut_seq.complete = false;

	spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq);



	poll_threads();

	CU_ASSERT_EQUAL(completed, 3);

	CU_ASSERT(ut_seq.complete);

	CU_ASSERT_EQUAL(ut_seq.status, 0);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1);

	CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1);

	CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u));

	CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0);

	CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0);

	g_seq_operations[ACCEL_OPC_CRC32C].count = 0;

	g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0;

	g_seq_operations[ACCEL_OPC_COPY].count = 0;



	for (i = 0; i < ACCEL_OPC_LAST; ++i) {

		g_modules_opc[i] = modules[i];

	}



	ut_clear_operations();

	spdk_put_io_channel(ioch);

	poll_threads();

}



static int

test_sequence_setup(void)

{
@@ -3854,6 +4086,7 @@ main(int argc, char **argv) 
#endif

	CU_ADD_TEST(seq_suite, test_sequence_driver);

	CU_ADD_TEST(seq_suite, test_sequence_same_iovs);

	CU_ADD_TEST(seq_suite, test_sequence_crc32);



	suite = CU_add_suite("accel", test_setup, test_cleanup);

	CU_ADD_TEST(suite, test_spdk_accel_task_complete);