Commit ac8071d6 authored by Krzysztof Sprzaczkowski's avatar Krzysztof Sprzaczkowski Committed by Tomasz Zawadzki
Browse files

lib/idxd: DIF check DSA implementation 



Extend the IDXD API with DIF Check operation support.

The DIF Check operation is used to check the correctness of
the Data Integrity Fields (DIF) field in the source data.
When performing a DIF Check operation, the device performs
the following actions on each block of source data and
the associated DIF:
- Calculate the Guard Tag and compare it to the Guard Tag
  field in the source DIF value.
- Verify the Application Tag and Reference Tag
  in the source DIF value.
- Update the Application Tag and Reference Tag for the next block
  of data, based on the DIF Flags.

Change-Id: I858700f8a370fdccd92115ebaa91def85540cb34
Signed-off-by: default avatarKrzysztof Sprzaczkowski <krzysztof.sprzaczkowski@intel.com>
Signed-off-by: default avatarSlawomir Ptak <slawomir.ptak@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/19779


Reviewed-by: default avatarShuhei Matsumoto <smatsumoto@nvidia.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 59456660

include/spdk/idxd.h

+25 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ 


#include "spdk/stdinc.h"

#include "spdk/idxd_spec.h"

#include "spdk/dif.h"



#ifdef __cplusplus

extern "C" {
@@ -302,6 +303,30 @@ int spdk_idxd_submit_decompress(struct spdk_idxd_io_channel *chan, 
				struct iovec *siov, uint32_t siovcnt,

				int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit a DIF check request

 *

 * This function will build the DIF check descriptor and then immediately submit

 * by writing to the proper device portal.

 *

 * \param chan IDXD channel to submit request.

 * \param siov Source iovec

 * \param siovcnt Number of elements in siov

 * \param num_blocks Total number of blocks to process

 * \param ctx DIF context. Contains the DIF configuration values, including the reference

 *            Application Tag value and initial value of the Reference Tag to check

 * \param flags Flags, optional flags that can vary per operation.

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the cb_arg parameter

 * in the completion callback.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_submit_dif_check(struct spdk_idxd_io_channel *chan,

			       struct iovec *siov, size_t siovcnt,

			       uint32_t num_blocks, const struct spdk_dif_ctx *ctx, int flags,

			       spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit an IDXD raw request.

 *

include/spdk/idxd_spec.h

+18 −0
Original line number Diff line number Diff line
@@ -38,6 +38,24 @@ extern "C" { 
#define IDXD_FLAG_DEST_STEERING_TAG	(1 << 15)

#define IDXD_FLAG_CRC_READ_CRC_SEED	(1 << 16)



#define IDXD_DSA_STATUS_DIF_ERROR	0x9



#define IDXD_DIF_FLAG_INVERT_CRC_RESULT		(1 << 3)

#define IDXD_DIF_FLAG_INVERT_CRC_SEED		(1 << 2)

#define IDXD_DIF_FLAG_DIF_BLOCK_SIZE_512	0x0

#define IDXD_DIF_FLAG_DIF_BLOCK_SIZE_520	0x1

#define IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4096	0x2

#define IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4104	0x3



#define IDXD_DIF_SOURCE_FLAG_SOURCE_REF_TAG_TYPE	(1 << 7)

#define IDXD_DIF_SOURCE_FLAG_REF_TAG_CHECK_DISABLE	(1 << 6)

#define IDXD_DIF_SOURCE_FLAG_GUARD_CHECK_DISABLE	(1 << 5)

#define IDXD_DIF_SOURCE_FLAG_SOURCE_APP_TAG_TYPE	(1 << 4)

#define IDXD_DIF_SOURCE_FLAG_APP_AND_REF_TAG_F_DETECT	(1 << 3)

#define IDXD_DIF_SOURCE_FLAG_APP_TAG_F_DETECT		(1 << 2)

#define IDXD_DIF_SOURCE_FLAG_ALL_F_DETECT		(1 << 1)

#define IDXD_DIF_SOURCE_FLAG_ENABLE_ALL_F_DETECT_ERR	(1)



#define IAA_FLAG_RD_SRC2_AECS		(1 << 16)

#define IAA_COMP_FLUSH_OUTPUT		(1 << 1)

#define IAA_COMP_APPEND_EOB		(1 << 2)

lib/idxd/idxd.c

+270 −0
Original line number Diff line number Diff line
@@ -25,6 +25,14 @@ 
/* The minimum number of entries in batch per flush */

#define IDXD_MIN_BATCH_FLUSH      32



#define DATA_BLOCK_SIZE_512 512

#define DATA_BLOCK_SIZE_520 520

#define DATA_BLOCK_SIZE_4096 4096

#define DATA_BLOCK_SIZE_4104 4104



#define METADATA_SIZE_8 8

#define METADATA_SIZE_16 16



static STAILQ_HEAD(, spdk_idxd_impl) g_idxd_impls = STAILQ_HEAD_INITIALIZER(g_idxd_impls);

static struct spdk_idxd_impl *g_idxd_impl;
@@ -1290,6 +1298,263 @@ spdk_idxd_submit_decompress(struct spdk_idxd_io_channel *chan, 
	return -EINVAL;

}



static inline int

idxd_get_dif_flags(const struct spdk_dif_ctx *ctx, uint8_t *flags)

{

	if (flags == NULL) {

		SPDK_ERRLOG("Flag should be non-null");

		return -EINVAL;

	}



	switch (ctx->guard_interval) {

	case DATA_BLOCK_SIZE_512:

		*flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_512;

		break;

	case DATA_BLOCK_SIZE_520:

		*flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_520;

		break;

	case DATA_BLOCK_SIZE_4096:

		*flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4096;

		break;

	case DATA_BLOCK_SIZE_4104:

		*flags = IDXD_DIF_FLAG_DIF_BLOCK_SIZE_4104;

		break;

	default:

		SPDK_ERRLOG("Invalid DIF block size %d\n", ctx->block_size - ctx->md_size);

		return -EINVAL;

	}



	return 0;

}



static inline int

idxd_get_source_dif_flags(const struct spdk_dif_ctx *ctx, uint8_t *flags)

{

	if (flags == NULL) {

		SPDK_ERRLOG("Flag should be non-null");

		return -EINVAL;

	}



	*flags = 0;



	if (!(ctx->dif_flags & SPDK_DIF_FLAGS_GUARD_CHECK)) {

		*flags |= IDXD_DIF_SOURCE_FLAG_GUARD_CHECK_DISABLE;

	}



	if (!(ctx->dif_flags & SPDK_DIF_FLAGS_REFTAG_CHECK)) {

		*flags |= IDXD_DIF_SOURCE_FLAG_REF_TAG_CHECK_DISABLE;

	}



	switch (ctx->dif_type) {

	case SPDK_DIF_TYPE1:

	case SPDK_DIF_TYPE2:

		/* If Type 1 or 2 is used, then all DIF checks are disabled when

		 * the Application Tag is 0xFFFF.

		 */

		*flags |= IDXD_DIF_SOURCE_FLAG_APP_TAG_F_DETECT;

		break;

	case SPDK_DIF_TYPE3:

		/* If Type 3 is used, then all DIF checks are disabled when the

		 * Application Tag is 0xFFFF and the Reference Tag is 0xFFFFFFFF

		 * (for PI 8 bytes format).

		 */

		*flags |= IDXD_DIF_SOURCE_FLAG_APP_AND_REF_TAG_F_DETECT;

		break;

	default:

		SPDK_ERRLOG("Invalid DIF type %d\n", ctx->dif_type);

		return -EINVAL;

	}

	return 0;

}



static inline int

idxd_get_app_tag_mask(const struct spdk_dif_ctx *ctx, uint16_t *app_tag_mask)

{

	if (!(ctx->dif_flags & SPDK_DIF_FLAGS_APPTAG_CHECK)) {

		/* The Source Application Tag Mask may be set to 0xffff

		 * to disable application tag checking */

		*app_tag_mask = 0xFFFF;

	} else {

		*app_tag_mask = ~ctx->apptag_mask;

	}

	return 0;

}



static inline int

idxd_validate_dif_common_params(const struct spdk_dif_ctx *ctx)

{

	/* Check byte offset from the start of the whole data buffer */

	if (ctx->data_offset != 0) {

		SPDK_ERRLOG("Byte offset from the start of the whole data buffer must be set to 0.");

		return -EINVAL;

	}



	/* Check seed value for guard computation */

	if (ctx->guard_seed != 0) {

		SPDK_ERRLOG("Seed value for guard computation must be set to 0.");

		return -EINVAL;

	}



	/* Check for supported metadata sizes */

	if (ctx->md_size != METADATA_SIZE_8 && ctx->md_size != METADATA_SIZE_16)  {

		SPDK_ERRLOG("Metadata size %d is not supported.\n", ctx->md_size);

		return -EINVAL;

	}



	/* Check for supported DIF PI formats */

	if (ctx->dif_pi_format != SPDK_DIF_PI_FORMAT_16) {

		SPDK_ERRLOG("DIF PI format %d is not supported.\n", ctx->dif_pi_format);

		return -EINVAL;

	}



	/* Check for supported metadata locations */

	if (ctx->md_interleave == false) {

		SPDK_ERRLOG("Separated metadata location is not supported.\n");

		return -EINVAL;

	}



	/* Check for supported DIF alignments */

	if (ctx->md_size == METADATA_SIZE_16 &&

	    (ctx->guard_interval == DATA_BLOCK_SIZE_512 ||

	     ctx->guard_interval == DATA_BLOCK_SIZE_4096)) {

		SPDK_ERRLOG("DIF left alignment in metadata is not supported.\n");

		return -EINVAL;

	}



	/* Check for supported DIF block sizes */

	if ((ctx->block_size - ctx->md_size) != DATA_BLOCK_SIZE_512 &&

	    (ctx->block_size - ctx->md_size) != DATA_BLOCK_SIZE_4096) {

		SPDK_ERRLOG("DIF block size %d is not supported.\n", ctx->block_size - ctx->md_size);

		return -EINVAL;

	}



	return 0;

}



static inline int

idxd_validate_dif_check_params(const struct spdk_dif_ctx *ctx)

{

	/* Validate common parameters */

	int rc = idxd_validate_dif_common_params(ctx);

	if (rc) {

		return rc;

	}



	return 0;

}



static inline int

idxd_validate_dif_check_buf_align(const struct spdk_dif_ctx *ctx, const uint64_t len)

{

	/* DSA can only process contiguous memory buffers, multiple of the block size */

	if (len % ctx->block_size != 0) {

		SPDK_ERRLOG("The memory buffer length (%ld) is not a multiple of block size with metadata (%d).\n",

			    len, ctx->block_size);

		return -EINVAL;

	}



	return 0;

}



int

spdk_idxd_submit_dif_check(struct spdk_idxd_io_channel *chan,

			   struct iovec *siov, size_t siovcnt,

			   uint32_t num_blocks, const struct spdk_dif_ctx *ctx, int flags,

			   spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_ops *first_op = NULL, *op = NULL;

	uint64_t src_seg_addr, src_seg_len;

	uint32_t num_blocks_done = 0;

	uint8_t dif_flags = 0, src_dif_flags = 0;

	uint16_t app_tag_mask = 0;

	int rc, count = 0;

	size_t i;



	assert(ctx != NULL);

	assert(chan != NULL);

	assert(siov != NULL);



	/* Validate DIF check parameters */

	rc = idxd_validate_dif_check_params(ctx);

	if (rc) {

		return rc;

	}



	/* Get DIF flags */

	rc = idxd_get_dif_flags(ctx, &dif_flags);

	if (rc) {

		return rc;

	}



	/* Get source DIF flags */

	rc = idxd_get_source_dif_flags(ctx, &src_dif_flags);

	if (rc) {

		return rc;

	}



	/* Get AppTag Mask */

	rc = idxd_get_app_tag_mask(ctx, &app_tag_mask);

	if (rc) {

		return rc;

	}



	rc = _idxd_setup_batch(chan);

	if (rc) {

		return rc;

	}



	for (i = 0; i < siovcnt; i++) {

		src_seg_addr = (uint64_t)siov[i].iov_base;

		src_seg_len = siov[i].iov_len;



		/* DSA processes the iovec buffers independently, so the buffers cannot

		 * be split (must be multiple of the block size) */



		/* Validate the memory buffer alignment */

		rc = idxd_validate_dif_check_buf_align(ctx, src_seg_len);

		if (rc) {

			goto error;

		}



		if (first_op == NULL) {

			rc = _idxd_prep_batch_cmd(chan, cb_fn, cb_arg, flags, &desc, &op);

			if (rc) {

				goto error;

			}



			first_op = op;

		} else {

			rc = _idxd_prep_batch_cmd(chan, NULL, NULL, flags, &desc, &op);

			if (rc) {

				goto error;

			}



			first_op->count++;

			op->parent = first_op;

		}



		count++;



		desc->opcode = IDXD_OPCODE_DIF_CHECK;

		desc->src_addr = src_seg_addr;

		desc->xfer_size = src_seg_len;

		desc->dif_chk.flags = dif_flags;

		desc->dif_chk.src_flags = src_dif_flags;

		desc->dif_chk.app_tag_seed = ctx->app_tag;

		desc->dif_chk.app_tag_mask = app_tag_mask;

		desc->dif_chk.ref_tag_seed = (uint32_t)ctx->init_ref_tag + num_blocks_done;



		num_blocks_done += (src_seg_len / ctx->block_size);

	}



	return _idxd_flush_batch(chan);



error:

	chan->batch->index -= count;

	return rc;

}



int

spdk_idxd_submit_raw_desc(struct spdk_idxd_io_channel *chan,

			  struct idxd_hw_desc *_desc,
@@ -1387,6 +1652,11 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) 
				*op->output_size = op->iaa_hw.output_size;

			}

			break;

		case IDXD_OPCODE_DIF_CHECK:

			if (spdk_unlikely(op->hw.status == IDXD_DSA_STATUS_DIF_ERROR)) {

				status = -EIO;

			}

			break;

		}



		/* TODO: WHAT IF THIS FAILED!? */

lib/idxd/spdk_idxd.map

+1 −0
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ 
	spdk_idxd_submit_fill;

	spdk_idxd_submit_compress;

	spdk_idxd_submit_decompress;

	spdk_idxd_submit_dif_check;

	spdk_idxd_submit_raw_desc;

	spdk_idxd_process_events;

	spdk_idxd_get_channel;

module/accel/dsa/accel_dsa.c

+27 −0
Original line number Diff line number Diff line
@@ -110,9 +110,27 @@ dsa_done(void *cb_arg, int status) 
{

	struct idxd_task *idxd_task = cb_arg;

	struct idxd_io_channel *chan;

	int rc;



	chan = idxd_task->chan;



	/* If the DSA DIF Check operation detects an error, detailed info about

	 * this error (like actual/expected values) needs to be obtained by

	 * calling the software DIF Verify operation.

	 */

	if (spdk_unlikely(status == -EIO)) {

		if (idxd_task->task.op_code == SPDK_ACCEL_OPC_DIF_VERIFY) {

			rc = spdk_dif_verify(idxd_task->task.s.iovs, idxd_task->task.s.iovcnt,

					     idxd_task->task.dif.num_blocks,

					     idxd_task->task.dif.ctx, idxd_task->task.dif.err);

			if (rc != 0) {

				SPDK_ERRLOG("DIF error detected. type=%d, offset=%" PRIu32 "\n",

					    idxd_task->task.dif.err->err_type,

					    idxd_task->task.dif.err->err_offset);

			}

		}

	}



	assert(chan->num_outstanding > 0);

	spdk_trace_record(TRACE_ACCEL_DSA_OP_COMPLETE, 0, 0, 0, chan->num_outstanding - 1);

	chan->num_outstanding--;
@@ -176,6 +194,12 @@ _process_single_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) 
						  task->seed, task->crc_dst, flags,

						  dsa_done, idxd_task);

		break;

	case SPDK_ACCEL_OPC_DIF_VERIFY:

		rc = spdk_idxd_submit_dif_check(chan->chan,

						task->s.iovs, task->s.iovcnt,

						task->dif.num_blocks, task->dif.ctx, flags,

						dsa_done, idxd_task);

		break;

	default:

		assert(false);

		rc = -EINVAL;
@@ -286,6 +310,9 @@ dsa_supports_opcode(enum spdk_accel_opcode opc) 
	case SPDK_ACCEL_OPC_CRC32C:

	case SPDK_ACCEL_OPC_COPY_CRC32C:

		return true;

	case SPDK_ACCEL_OPC_DIF_VERIFY:

		/* Supported only if the IOMMU is enabled */

		return spdk_iommu_is_enabled();

	default:

		return false;

	}