Commit 0ff560ea authored by paul luse's avatar paul luse Committed by Jim Harris
Browse files

lib/idxd: Add compress/decompress support to low level lib 



Accel module coming in next patch...

Add support for compress and decompress. The low level IDXD
library supports both DSA and IAA hardware.  There are separate
modules for DSA and IAA.

accel_perf patch follows.

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: I55014122f6555f80985c11d49a54eddc5d51c337
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12292


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
parent 3ac967ba

CHANGELOG.md

+5 −0
Original line number Diff line number Diff line
@@ -45,6 +45,9 @@ The RPC `idxd_scan_accel_engine` has been renamed to `dsa_scan_accel_engine` 
Many HW related structs/functions with the name `idxd` have been renamed `dsa`

to more accurately represent the HW they are associated with.



Two new functions were added to the library `spdk_idxd_submit_compress` and

`spdk_idxd_submit_decompress`



### accel_fw



A new parameter `flags` was added to accel API.
@@ -54,6 +57,8 @@ The APIs include: 
`spdk_accel_submit_fill`

`spdk_accel_submit_copy_crc32c`

`spdk_accel_submit_copy_crc32cv`

`spdk_accel_submit_compress`

`spdk_accel_submit_decompress`



A new flag `ACCEL_FLAG_PERSISTENT` was added to indicate the target memory is PMEM.

include/spdk/idxd.h

+47 −0
Original line number Diff line number Diff line
@@ -286,6 +286,53 @@ int spdk_idxd_submit_copy_crc32c(struct spdk_idxd_io_channel *chan, 
				 uint32_t seed, uint32_t *crc_dst, int flags,

				 spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit an IAA memory compress request.

 *

 * This function will build the compress descriptor and then immediately submit

 * by writing to the proper device portal.

 *

 * \param chan IDXD channel to submit request.

 * \param diov Destination iovec. diov with diovcnt must be large enough to hold compressed data.

 * \param diovcnt Number of elements in diov for decompress buffer.

 * \param siov Source iovec

 * \param siovcnt Number of elements in siov

 * \param output_size The size of the compressed data

 * \param flags Flags, optional flags that can vary per operation.

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the arg parameter in

 * the completion callback.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_submit_compress(struct spdk_idxd_io_channel *chan,

			      struct iovec *diov, uint32_t diovcnt,

			      struct iovec *siov, uint32_t siovcnt, uint32_t *output_size,

			      int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Build and submit an IAA memory decompress request.

 *

 * This function will build the decompress descriptor and then immediately submit

 * by writing to the proper device portal.

 *

 * \param chan IDXD channel to submit request.

 * \param diov Destination iovec. diov with diovcnt must be large enough to hold decompressed data.

 * \param diovcnt Number of elements in diov for decompress buffer.

 * \param siov Source iovec

 * \param siovcnt Number of elements in siov

 * \param flags Flags, optional flags that can vary per operation.

 * \param cb_fn Callback function which will be called when the request is complete.

 * \param cb_arg Opaque value which will be passed back as the arg parameter in

 * the completion callback.

 *

 * \return 0 on success, negative errno on failure.

 */

int spdk_idxd_submit_decompress(struct spdk_idxd_io_channel *chan,

				struct iovec *diov, uint32_t diovcnt,

				struct iovec *siov, uint32_t siovcnt,

				int flags, spdk_idxd_req_cb cb_fn, void *cb_arg);



/**

 * Check for completed requests on an IDXD channel.

 *

include/spdk/idxd_spec.h

+19 −0
Original line number Diff line number Diff line
@@ -65,6 +65,11 @@ extern "C" { 
#define IDXD_FLAG_DEST_STEERING_TAG	(1 << 15)

#define IDXD_FLAG_CRC_READ_CRC_SEED	(1 << 16)



#define IAA_FLAG_RD_SRC2_AECS		(1 << 16)

#define IAA_COMP_FLUSH_OUTPUT		(1 << 1)

#define IAA_COMP_APPEND_EOB		(1 << 2)

#define IAA_COMP_FLAGS			(IAA_COMP_FLUSH_OUTPUT | IAA_COMP_APPEND_EOB)



/*

 * IDXD is a family of devices, DSA and IAA.

 */
@@ -343,6 +348,20 @@ struct iaa_hw_comp_record { 
};

SPDK_STATIC_ASSERT(sizeof(struct iaa_hw_comp_record) == 64, "size mismatch");



struct iaa_aecs {

	uint32_t crc;

	uint32_t xor_checksum;

	uint32_t rsvd[5];

	uint32_t num_output_accum_bits;

	uint8_t output_accum[256];

	uint32_t ll_sym[286];

	uint32_t rsvd1;

	uint32_t rsvd3;

	uint32_t d_sym[30];

	uint32_t pad[2];

};

SPDK_STATIC_ASSERT(sizeof(struct iaa_aecs) == 1568, "size mismatch");



union idxd_gencap_register {

	struct {

		uint64_t block_on_fault: 1;

include/spdk_internal/idxd.h

+2 −0
Original line number Diff line number Diff line
@@ -63,6 +63,8 @@ enum dsa_opcode { 
	IDXD_OPCODE_DIF_STRP	= 20,

	IDXD_OPCODE_DIF_UPDT	= 21,

	IDXD_OPCODE_CFLUSH	= 32,

	IDXD_OPCODE_DECOMPRESS	= 66,

	IDXD_OPCODE_COMPRESS	= 67,

};



#ifdef __cplusplus

lib/idxd/idxd.c

+177 −21
Original line number Diff line number Diff line
@@ -218,7 +218,8 @@ spdk_idxd_get_channel(struct spdk_idxd_device *idxd) 
	struct spdk_idxd_io_channel *chan;

	struct idxd_hw_desc *desc;

	struct idxd_ops *op;

	int i, num_descriptors, rc;

	int i, num_descriptors, rc = -1;

	uint32_t comp_rec_size;



	assert(idxd != NULL);
@@ -238,7 +239,8 @@ spdk_idxd_get_channel(struct spdk_idxd_device *idxd) 
	if (idxd->num_channels == idxd->chan_per_device) {

		/* too many channels sharing this device */

		pthread_mutex_unlock(&idxd->num_channels_lock);

		goto err_chan;

		SPDK_ERRLOG("Too many channels sharing this device\n");

		goto error;

	}



	/* Have each channel start at a different offset. */
@@ -254,39 +256,46 @@ spdk_idxd_get_channel(struct spdk_idxd_device *idxd) 
					      0x40, NULL,

					      SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);

	if (chan->desc_base == NULL) {

		SPDK_ERRLOG("Failed to allocate descriptor memory\n");

		goto err_chan;

		SPDK_ERRLOG("Failed to allocate DSA descriptor memory\n");

		goto error;

	}



	chan->ops_base = op = spdk_zmalloc(num_descriptors * sizeof(struct idxd_ops),

					   0x40, NULL,

					   SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);

	if (chan->ops_base == NULL) {

		SPDK_ERRLOG("Failed to allocate completion memory\n");

		goto err_op;

		SPDK_ERRLOG("Failed to allocate idxd_ops memory\n");

		goto error;

	}



	if (idxd->type == IDXD_DEV_TYPE_DSA) {

		comp_rec_size = sizeof(struct dsa_hw_comp_record);

		if (_dsa_alloc_batches(chan, num_descriptors)) {

			goto error;

		}

	} else {

		comp_rec_size = sizeof(struct iaa_hw_comp_record);

	}



	for (i = 0; i < num_descriptors; i++) {

		STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);

		op->desc = desc;

		rc = _vtophys(&op->hw, &desc->completion_addr, sizeof(struct dsa_hw_comp_record));

		rc = _vtophys(&op->hw, &desc->completion_addr, comp_rec_size);

		if (rc) {

			SPDK_ERRLOG("Failed to translate completion memory\n");

			goto err_op;

			goto error;

		}

		op++;

		desc++;

	}



	if (_dsa_alloc_batches(chan, num_descriptors)) {

		return NULL;

	}



	return chan;

err_op:



error:

	spdk_free(chan->ops_base);

	chan->ops_base = NULL;

	spdk_free(chan->desc_base);

	chan->desc_base = NULL;

err_chan:

	free(chan);

	return NULL;

}
@@ -299,6 +308,7 @@ spdk_idxd_put_channel(struct spdk_idxd_io_channel *chan) 
	struct idxd_batch *batch;



	assert(chan != NULL);

	assert(chan->idxd != NULL);



	if (chan->batch) {

		idxd_batch_cancel(chan, -ECANCELED);
@@ -338,18 +348,21 @@ idxd_get_impl_by_name(const char *impl_name) 
void

spdk_idxd_set_config(bool kernel_mode)

{

	if (g_idxd_impl != NULL) {

		SPDK_ERRLOG("Cannot change idxd implementation after devices are initialized\n");

		assert(false);

		return;

	}

	struct spdk_idxd_impl *tmp;



	if (kernel_mode) {

		g_idxd_impl = idxd_get_impl_by_name(KERNEL_DRIVER_NAME);

		tmp = idxd_get_impl_by_name(KERNEL_DRIVER_NAME);

	} else {

		g_idxd_impl = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME);

		tmp = idxd_get_impl_by_name(USERSPACE_DRIVER_NAME);

	}



	if (g_idxd_impl != NULL && g_idxd_impl != tmp) {

		SPDK_ERRLOG("Cannot change idxd implementation after devices are initialized\n");

		assert(false);

		return;

	}

	g_idxd_impl = tmp;



	if (g_idxd_impl == NULL) {

		SPDK_ERRLOG("Cannot set the idxd implementation with %s mode\n",

			    kernel_mode ? KERNEL_DRIVER_NAME : USERSPACE_DRIVER_NAME);
@@ -1125,6 +1138,142 @@ error: 
	return rc;

}



static inline int

_idxd_submit_compress_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,

			     uint64_t nbytes_dst, uint64_t nbytes_src, uint32_t *output_size,

			     int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_ops *op;

	uint64_t src_addr, dst_addr;

	int rc;



	/* Common prep. */

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);

	if (rc) {

		return rc;

	}



	rc = _vtophys(src, &src_addr, nbytes_src);

	if (rc) {

		goto error;

	}



	rc = _vtophys(dst, &dst_addr, nbytes_dst);

	if (rc) {

		goto error;

	}



	/* Command specific. */

	desc->opcode = IDXD_OPCODE_COMPRESS;

	desc->src1_addr = src_addr;

	desc->dst_addr = dst_addr;

	desc->src1_size = nbytes_src;

	desc->iaa.max_dst_size = nbytes_dst;

	desc->iaa.src2_size = sizeof(struct iaa_aecs);

	desc->iaa.src2_addr = (uint64_t)chan->idxd->aecs;

	desc->flags |= IAA_FLAG_RD_SRC2_AECS;

	desc->compr_flags = IAA_COMP_FLAGS;

	op->output_size = output_size;



	_submit_to_hw(chan, op);

	return 0;

error:

	STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);

	return rc;

}



int

spdk_idxd_submit_compress(struct spdk_idxd_io_channel *chan,

			  struct iovec *diov, uint32_t diovcnt,

			  struct iovec *siov, uint32_t siovcnt, uint32_t *output_size,

			  int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	assert(chan != NULL);

	assert(diov != NULL);

	assert(siov != NULL);



	if (diovcnt == 1 && siovcnt == 1) {

		/* Simple case - copying one buffer to another */

		if (diov[0].iov_len < siov[0].iov_len) {

			return -EINVAL;

		}



		return _idxd_submit_compress_single(chan, diov[0].iov_base, siov[0].iov_base,

						    diov[0].iov_len, siov[0].iov_len,

						    output_size, flags, cb_fn, cb_arg);

	}

	/* TODO: vectored support */

	return -EINVAL;

}



static inline int

_idxd_submit_decompress_single(struct spdk_idxd_io_channel *chan, void *dst, const void *src,

			       uint64_t nbytes_dst, uint64_t nbytes, int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_ops *op;

	uint64_t src_addr, dst_addr;

	int rc;



	/* Common prep. */

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, flags, &desc, &op);

	if (rc) {

		return rc;

	}



	rc = _vtophys(src, &src_addr, nbytes);

	if (rc) {

		goto error;

	}



	rc = _vtophys(dst, &dst_addr, nbytes_dst);

	if (rc) {

		goto error;

	}



	/* Command specific. */

	desc->opcode = IDXD_OPCODE_COMPRESS;

	desc->src1_addr = src_addr;

	desc->dst_addr = dst_addr;

	desc->src1_size = nbytes;

	desc->iaa.max_dst_size = nbytes_dst;

	desc->iaa.src2_size = sizeof(struct iaa_aecs);

	desc->iaa.src2_addr = (uint64_t)chan->idxd->aecs;

	desc->flags |= IAA_FLAG_RD_SRC2_AECS;

	desc->compr_flags = IAA_COMP_FLAGS;



	_submit_to_hw(chan, op);

	return 0;

error:

	STAILQ_INSERT_TAIL(&chan->ops_pool, op, link);

	return rc;

}



int

spdk_idxd_submit_decompress(struct spdk_idxd_io_channel *chan,

			    struct iovec *diov, uint32_t diovcnt,

			    struct iovec *siov, uint32_t siovcnt,

			    int flags, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	assert(chan != NULL);

	assert(diov != NULL);

	assert(siov != NULL);



	if (diovcnt == 1 && siovcnt == 1) {

		/* Simple case - copying one buffer to another */

		if (diov[0].iov_len < siov[0].iov_len) {

			return -EINVAL;

		}



		return _idxd_submit_decompress_single(chan, diov[0].iov_base, siov[0].iov_base,

						      diov[0].iov_len, siov[0].iov_len,

						      flags, cb_fn, cb_arg);

	}

	/* TODO: vectored support */

	return -EINVAL;

}



static inline void

_dump_sw_error_reg(struct spdk_idxd_io_channel *chan)

{
@@ -1162,7 +1311,9 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) 
		STAILQ_REMOVE_HEAD(&chan->ops_outstanding, link);

		rc++;



		/* Status is in the same location for both IAA and DSA completion records. */

		if (spdk_unlikely(IDXD_FAILURE(op->hw.status))) {

			SPDK_ERRLOG("Completion status 0x%x\n", op->hw.status);

			status = -EINVAL;

			_dump_sw_error_reg(chan);

		}
@@ -1183,6 +1334,11 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) 
				status = op->hw.result;

			}

			break;

		case IDXD_OPCODE_COMPRESS:

			if (spdk_likely(status == 0 && op->output_size != NULL)) {

				*op->output_size = op->iaa_hw.output_size;

			}

			break;

		}



		/* TODO: WHAT IF THIS FAILED!? */