Commit 92fb2251 authored by Shuhei Matsumoto's avatar Shuhei Matsumoto Committed by Konrad Sztyber
Browse files

dif: dif_generate/verify_copy() supports NVMe PRACT = 1 and MD size > PI size 



In the NVMe spec, if PRACT is 1 and metadata size is larger than
PI size, PI is just overwritten for write and PI is just verified for
read. spdk_dif_generate_copy() and spdk_dif_verify_copy() simulate this
behavior.

Update unit test cases for verification.

Signed-off-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Change-Id: I0e26a0c2675af416d9d181e3cb4ecb084939c6a4
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/23688


Reviewed-by: default avatarAleksey Marchuk <alexeymar@nvidia.com>
Reviewed-by: default avatarJim Harris <jim.harris@nvidia.com>
Community-CI: Mellanox Build Bot
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Community CI Samsung <spdk.community.ci.samsung@gmail.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 79daf868

include/spdk/dif.h

+8 −0
Original line number Diff line number Diff line
@@ -224,6 +224,10 @@ int spdk_dif_update_crc32c(struct iovec *iovs, int iovcnt, uint32_t num_blocks, 
/**

 * Copy data and generate DIF for extended LBA payload.

 *

 * NOTE: If PRACT is set in the DIF context, this function simulates the NVMe PRACT feature.

 * If metadata size is larger than DIF size, not only bounce buffer but also source buffer

 * should be extended LBA payload.



 * \param iovs iovec array describing the LBA payload.

 * \param iovcnt Number of elements in the iovec array.

 * \param bounce_iovs A contiguous buffer forming extended LBA payload.
@@ -240,6 +244,10 @@ int spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_ 
/**

 * Verify DIF and copy data for extended LBA payload.

 *

 * NOTE: If PRACT is set in the DIF context, this function simulates the NVMe PRACT feature.

 * If metadata size is larger than DIF size, not only bounce buffer but also destination buffer

 * should be extended LBA payload.

 *

 * \param iovs iovec array describing the LBA payload.

 * \param iovcnt Number of elements in the iovec array.

 * \param bounce_iovs A contiguous buffer forming extended LBA payload.

lib/util/dif.c

+210 −2
Original line number Diff line number Diff line
@@ -1295,6 +1295,103 @@ _spdk_dif_insert_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 
	return 0;

}



static void

_dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		    uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)

{

	uint8_t *src, *dst;

	uint64_t guard = 0;



	_dif_sgl_get_buf(src_sgl, &src, NULL);

	_dif_sgl_get_buf(dst_sgl, &dst, NULL);



	if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {

		guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval,

						 ctx->dif_pi_format);

	} else {

		memcpy(dst, src, ctx->guard_interval);

	}



	_dif_generate(dst + ctx->guard_interval, guard, offset_blocks, ctx);



	_dif_sgl_advance(src_sgl, ctx->block_size);

	_dif_sgl_advance(dst_sgl, ctx->block_size);

}



static void

dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		   uint32_t num_blocks, const struct spdk_dif_ctx *ctx)

{

	uint32_t offset_blocks;



	for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {

		_dif_overwrite_copy(src_sgl, dst_sgl, offset_blocks, ctx);

	}

}



static void

_dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

			  uint32_t offset_blocks, const struct spdk_dif_ctx *ctx)

{

	uint64_t guard = 0;

	struct spdk_dif dif = {};



	if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {

		guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl,

						       ctx->guard_interval, ctx->dif_pi_format);

	} else {

		_data_copy_split(dst_sgl, src_sgl, ctx->guard_interval);

	}



	_dif_sgl_advance(src_sgl, ctx->block_size - ctx->guard_interval);



	_dif_generate(&dif, guard, offset_blocks, ctx);

	dif_store_split(dst_sgl, &dif, ctx);

}



static void

dif_overwrite_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

			 uint32_t num_blocks, const struct spdk_dif_ctx *ctx)

{

	uint32_t offset_blocks;



	for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {

		_dif_overwrite_copy_split(src_sgl, dst_sgl, offset_blocks, ctx);

	}

}



static void

dif_disable_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		 uint32_t num_blocks, const struct spdk_dif_ctx *ctx)

{

	_data_copy_split(dst_sgl, src_sgl, ctx->block_size * num_blocks);

}



static int

_spdk_dif_overwrite_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

			 uint32_t num_blocks, const struct spdk_dif_ctx *ctx)

{

	if (!_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks) ||

	    !_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks)) {

		SPDK_ERRLOG("Size of iovec arrays are not valid.\n");

		return -EINVAL;

	}



	if (_dif_is_disabled(ctx->dif_type)) {

		dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx);

		return 0;

	}



	if (_dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size) &&

	    _dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size)) {

		dif_overwrite_copy(src_sgl, dst_sgl, num_blocks, ctx);

	} else {

		dif_overwrite_copy_split(src_sgl, dst_sgl, num_blocks, ctx);

	}



	return 0;

}



int

spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs,

		       int bounce_iovcnt, uint32_t num_blocks,
@@ -1309,7 +1406,7 @@ spdk_dif_generate_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs 
	    ctx->md_size == _dif_size(ctx->dif_pi_format)) {

		return _spdk_dif_insert_copy(&src_sgl, &dst_sgl, num_blocks, ctx);

	} else {

		return -ENOTSUP;

		return _spdk_dif_overwrite_copy(&src_sgl, &dst_sgl, num_blocks, ctx);

	}

}
@@ -1474,6 +1571,117 @@ _spdk_dif_strip_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl, 
	}

}



static int

_dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		 uint32_t offset_blocks, const struct spdk_dif_ctx *ctx,

		 struct spdk_dif_error *err_blk)

{

	uint8_t *src, *dst;

	int rc;

	uint64_t guard = 0;



	_dif_sgl_get_buf(src_sgl, &src, NULL);

	_dif_sgl_get_buf(dst_sgl, &dst, NULL);



	if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {

		guard = _dif_generate_guard_copy(ctx->guard_seed, dst, src, ctx->guard_interval,

						 ctx->dif_pi_format);

	} else {

		memcpy(dst, src, ctx->guard_interval);

	}



	rc = _dif_verify(src + ctx->guard_interval, guard, offset_blocks, ctx, err_blk);

	if (rc != 0) {

		return rc;

	}



	_dif_sgl_advance(src_sgl, ctx->block_size);

	_dif_sgl_advance(dst_sgl, ctx->block_size);



	return 0;

}



static int

dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

		struct spdk_dif_error *err_blk)

{

	uint32_t offset_blocks;

	int rc;



	for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {

		rc = _dif_verify_copy(src_sgl, dst_sgl, offset_blocks, ctx, err_blk);

		if (rc != 0) {

			return rc;

		}

	}



	return 0;

}



static int

_dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		       uint32_t offset_blocks, const struct spdk_dif_ctx *ctx,

		       struct spdk_dif_error *err_blk)

{

	uint64_t guard = 0;

	struct spdk_dif dif = {};



	if (ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK) {

		guard = _dif_generate_guard_copy_split(ctx->guard_seed, dst_sgl, src_sgl,

						       ctx->guard_interval, ctx->dif_pi_format);

	} else {

		_data_copy_split(dst_sgl, src_sgl, ctx->guard_interval);

	}



	dif_load_split(src_sgl, &dif, ctx);

	_dif_sgl_advance(dst_sgl, ctx->block_size - ctx->guard_interval);



	return _dif_verify(&dif, guard, offset_blocks, ctx, err_blk);

}



static int

dif_verify_copy_split(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		      uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

		      struct spdk_dif_error *err_blk)

{

	uint32_t offset_blocks;

	int rc;



	for (offset_blocks = 0; offset_blocks < num_blocks; offset_blocks++) {

		rc = _dif_verify_copy_split(src_sgl, dst_sgl, offset_blocks, ctx, err_blk);

		if (rc != 0) {

			return rc;

		}

	}



	return 0;

}



static int

_spdk_dif_verify_copy(struct _dif_sgl *src_sgl, struct _dif_sgl *dst_sgl,

		      uint32_t num_blocks, const struct spdk_dif_ctx *ctx,

		      struct spdk_dif_error *err_blk)

{

	if (!_dif_sgl_is_valid(dst_sgl, ctx->block_size * num_blocks) ||

	    !_dif_sgl_is_valid(src_sgl, ctx->block_size * num_blocks)) {

		SPDK_ERRLOG("Size of iovec arrays are not valid\n");

		return -EINVAL;

	}



	if (_dif_is_disabled(ctx->dif_type)) {

		dif_disable_copy(src_sgl, dst_sgl, num_blocks, ctx);

		return 0;

	}



	if (_dif_sgl_is_bytes_multiple(dst_sgl, ctx->block_size) &&

	    _dif_sgl_is_bytes_multiple(src_sgl, ctx->block_size)) {

		return dif_verify_copy(src_sgl, dst_sgl, num_blocks, ctx, err_blk);

	} else {

		return dif_verify_copy_split(src_sgl, dst_sgl, num_blocks, ctx, err_blk);

	}

}



int

spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs,

		     int bounce_iovcnt, uint32_t num_blocks,
@@ -1489,7 +1697,7 @@ spdk_dif_verify_copy(struct iovec *iovs, int iovcnt, struct iovec *bounce_iovs, 
	    ctx->md_size == _dif_size(ctx->dif_pi_format)) {

		return _spdk_dif_strip_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk);

	} else {

		return -ENOTSUP;

		return _spdk_dif_verify_copy(&src_sgl, &dst_sgl, num_blocks, ctx, err_blk);

	}

}

test/unit/lib/util/dif.c/dif_ut.c

+119 −2
Original line number Diff line number Diff line
@@ -1587,7 +1587,11 @@ dif_copy_gen_and_verify(struct iovec *iovs, int iovcnt, 
	int rc;

	struct spdk_dif_ctx_init_ext_opts dif_opts;



	if (dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) {

		rc = ut_data_pattern_generate(iovs, iovcnt, block_size, md_size, num_blocks);

	} else {

		rc = ut_data_pattern_generate(iovs, iovcnt, block_size - md_size, 0, num_blocks);

	}

	CU_ASSERT(rc == 0);



	dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format);
@@ -1599,10 +1603,17 @@ dif_copy_gen_and_verify(struct iovec *iovs, int iovcnt, 
	rc = spdk_dif_generate_copy(iovs, iovcnt, bounce_iovs, bounce_iovcnt, num_blocks, &ctx);

	CU_ASSERT(rc == 0);



	rc = ut_data_pattern_verify(bounce_iovs, bounce_iovcnt, block_size, md_size, num_blocks);

	CU_ASSERT(rc == 0);



	rc = spdk_dif_verify_copy(iovs, iovcnt, bounce_iovs, bounce_iovcnt, num_blocks, &ctx, NULL);

	CU_ASSERT(rc == 0);



	if (dif_flags & SPDK_DIF_FLAGS_NVME_PRACT) {

		rc = ut_data_pattern_verify(iovs, iovcnt, block_size, md_size, num_blocks);

	} else {

		rc = ut_data_pattern_verify(iovs, iovcnt, block_size - md_size, 0, num_blocks);

	}

	CU_ASSERT(rc == 0);

}
@@ -1797,6 +1808,75 @@ dif_copy_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(void) 
	_dif_copy_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(SPDK_DIF_PI_FORMAT_64);

}



static void

_nvme_pract_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(

	enum spdk_dif_pi_format dif_pi_format)

{

	struct iovec iovs[4], bounce_iovs[2];

	int i, num_blocks;



	num_blocks = 0;



	for (i = 0; i < 4; i++) {

		_iov_alloc_buf(&iovs[i], (4096 + 128) * (i + 1));

		num_blocks += i + 1;

	}



	num_blocks = 0;



	for (i = 0; i < 2; i++) {

		_iov_alloc_buf(&bounce_iovs[i], (4096 + 128) * 2 * (i + 1));

		num_blocks += 2 * (i + 1);

	}



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				false, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				false, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				false, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				false, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_REFTAG_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				true, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				true, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				true, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_APPTAG_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	dif_copy_gen_and_verify(iovs, 4, bounce_iovs, 2, 4096 + 128, 128, num_blocks,

				true, SPDK_DIF_TYPE1, SPDK_DIF_FLAGS_REFTAG_CHECK | SPDK_DIF_FLAGS_NVME_PRACT,

				22, 0xFFFF, 0x22, dif_pi_format);



	for (i = 0; i < 4; i++) {

		_iov_free_buf(&iovs[i]);

	}



	for (i = 0; i < 2; i++) {

		_iov_free_buf(&bounce_iovs[i]);

	}

}



static void

nvme_pract_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(void)

{

	_nvme_pract_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(SPDK_DIF_PI_FORMAT_32);

	_nvme_pract_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test(SPDK_DIF_PI_FORMAT_64);

}



static void

dif_copy_sec_4096_md_128_prchk_7_multi_iovs(void)

{
@@ -2089,6 +2169,41 @@ dif_copy_sec_4096_md_128_prchk_7_multi_iovs_complex_splits_test(void) 
	_iov_free_buf(&bounce_iov);

}



static void

nvme_pract_sec_4096_md_128_prchk_7_multi_iovs_complex_splits_test(void)

{

	struct iovec iovs[4], bounce_iov;

	uint32_t dif_flags;

	int i;



	dif_flags = SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK |

		    SPDK_DIF_FLAGS_REFTAG_CHECK | SPDK_DIF_FLAGS_NVME_PRACT;



	/* data[0][2047:0] */

	_iov_alloc_buf(&iovs[0], 2048);



	/* data[0][4223:2048], data[1][4220:0] */

	_iov_alloc_buf(&iovs[1], 2176 + 4221);



	/* data[1][4223:4221] data[2][4210:0] */

	_iov_alloc_buf(&iovs[2], 3 + 4211);



	/* data[2][4223:4211], data[3][4223:0] */

	_iov_alloc_buf(&iovs[3], 13 + 4224);



	_iov_alloc_buf(&bounce_iov, (4096 + 128) * 4);



	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 1, 4096 + 128, 128, 4,

				false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22, SPDK_DIF_PI_FORMAT_32);

	dif_copy_gen_and_verify(iovs, 4, &bounce_iov, 1, 4096 + 128, 128, 4,

				false, SPDK_DIF_TYPE1, dif_flags, 22, 0xFFFF, 0x22, SPDK_DIF_PI_FORMAT_64);



	for (i = 0; i < 4; i++) {

		_iov_free_buf(&iovs[i]);

	}

	_iov_free_buf(&bounce_iov);

}



static void

_dif_copy_inject_error_and_verify(struct iovec *iovs, int iovcnt, struct iovec *bounce_iov,

				  uint32_t block_size, uint32_t md_size, uint32_t num_blocks,
@@ -4462,6 +4577,7 @@ main(int argc, char **argv) 
	CU_ADD_TEST(suite, dif_copy_sec_512_md_8_prchk_0_1_2_4_multi_iovs);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_prchk_0_1_2_4_multi_iovs_test);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test);

	CU_ADD_TEST(suite, nvme_pract_sec_4096_md_128_prchk_0_1_2_4_multi_bounce_iovs_test);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_prchk_7_multi_iovs);

	CU_ADD_TEST(suite, dif_copy_sec_512_md_8_prchk_7_multi_iovs_split_data);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_prchk_7_multi_iovs_split_data_test);
@@ -4469,6 +4585,7 @@ main(int argc, char **argv) 
	CU_ADD_TEST(suite, dif_copy_sec_512_md_8_prchk_7_multi_bounce_iovs_complex_splits);

	CU_ADD_TEST(suite, dif_copy_sec_512_md_8_dif_disable_multi_bounce_iovs_complex_splits);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_prchk_7_multi_iovs_complex_splits_test);

	CU_ADD_TEST(suite, nvme_pract_sec_4096_md_128_prchk_7_multi_iovs_complex_splits_test);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_inject_1_2_4_8_multi_iovs_test);

	CU_ADD_TEST(suite, dif_copy_sec_4096_md_128_inject_1_2_4_8_multi_iovs_split_test);

	CU_ADD_TEST(suite, dix_sec_0_md_8_error);