Commit eb5a1113 authored by paul luse's avatar paul luse Committed by Jim Harris
Browse files

lib/idxd: move the crc destination * from the desc to comp record 



Was using reserved field in CRC to store the final address of where
to put the result, this not legal.  Move to the completion record
and slightly re-arrange the struct to keep it at 96 bytes.

Refacorted the IO prep function so the caller can udpate both the
descriptor and completion records instead of continuing to add
parameters to the prep function for opcodes that need something
unique in the completion record.  This also allowed for a minor
fix where the prep function was returning NULL when vtophys failed
which would have indicated busy as opposed to failire.  Now we can
proprely fail that path.

fixes #1929

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: Ic23bc7b68bdd5757c30b7963880677f423368e20
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/7735


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 274f52b5

lib/idxd/idxd.c

+36 −35
Original line number Diff line number Diff line
@@ -406,9 +406,9 @@ _vtophys(const void *buf, uint64_t *buf_addr, uint64_t size) 
	return 0;

}



static struct idxd_hw_desc *

static int

_idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn,

		   void *cb_arg, struct idxd_batch *batch)

		   void *cb_arg, struct idxd_hw_desc **_desc, struct idxd_comp **_comp)

{

	uint32_t index;

	struct idxd_hw_desc *desc;
@@ -419,18 +419,18 @@ _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, 
	index = spdk_bit_array_find_first_clear(chan->ring_slots, 0);

	if (index == UINT32_MAX) {

		/* ran out of ring slots */

		return NULL;

		return -EBUSY;

	}



	spdk_bit_array_set(chan->ring_slots, index);



	desc = &chan->desc[index];

	comp = &chan->completions[index];

	desc = *_desc = &chan->desc[index];

	comp = *_comp = &chan->completions[index];



	rc = _vtophys(&comp->hw, &comp_hw_addr, sizeof(struct idxd_hw_comp_record));

	if (rc) {

		spdk_bit_array_clear(chan->ring_slots, index);

		return NULL;

		return rc;

	}



	_track_comp(chan, false, index, comp, desc, NULL);
@@ -439,9 +439,8 @@ _idxd_prep_command(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, 
	desc->completion_addr = comp_hw_addr;

	comp->cb_arg = cb_arg;

	comp->cb_fn = cb_fn;

	comp->batch = batch;



	return desc;

	return 0;

}



int
@@ -449,13 +448,14 @@ spdk_idxd_submit_copy(struct spdk_idxd_io_channel *chan, void *dst, const void * 
		      uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_comp *comp;

	uint64_t src_addr, dst_addr;

	int rc;



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, NULL);

	if (desc == NULL) {

		return -EBUSY;

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(src, &src_addr, nbytes);
@@ -486,6 +486,7 @@ spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *d 
			  const void *src, uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_comp *comp;

	uint64_t src_addr, dst1_addr, dst2_addr;

	int rc;
@@ -495,9 +496,9 @@ spdk_idxd_submit_dualcast(struct spdk_idxd_io_channel *chan, void *dst1, void *d 
	}



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, NULL);

	if (desc == NULL) {

		return -EBUSY;

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(src, &src_addr, nbytes);
@@ -533,13 +534,14 @@ spdk_idxd_submit_compare(struct spdk_idxd_io_channel *chan, void *src1, const vo 
			 uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_comp *comp;

	uint64_t src1_addr, src2_addr;

	int rc;



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, NULL);

	if (desc == NULL) {

		return -EBUSY;

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(src1, &src1_addr, nbytes);
@@ -569,13 +571,14 @@ spdk_idxd_submit_fill(struct spdk_idxd_io_channel *chan, void *dst, uint64_t fil 
		      uint64_t nbytes, spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_comp *comp;

	uint64_t dst_addr;

	int rc;



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, NULL);

	if (desc == NULL) {

		return -EBUSY;

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(dst, &dst_addr, nbytes);
@@ -601,32 +604,29 @@ spdk_idxd_submit_crc32c(struct spdk_idxd_io_channel *chan, uint32_t *dst, void * 
			spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	uint64_t src_addr, dst_addr;

	struct idxd_comp *comp;

	uint64_t src_addr;

	int rc;



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, NULL);

	if (desc == NULL) {

		return -EBUSY;

	}



	rc = _vtophys(src, &src_addr, nbytes);

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(dst, &dst_addr, nbytes);

	rc = _vtophys(src, &src_addr, nbytes);

	if (rc) {

		return rc;

	}



	/* Command specific. */

	desc->opcode = IDXD_OPCODE_CRC32C_GEN;

	desc->dst_addr = dst_addr;

	desc->dst_addr = 0; /* Per spec, needs to be clear. */

	desc->src_addr = src_addr;

	desc->flags &= IDXD_CLEAR_CRC_FLAGS;

	desc->crc32c.seed = seed;

	desc->xfer_size = nbytes;

	comp->crc_dst = dst;



	/* Submit operation. */

	movdir64b(chan->portal, desc);
@@ -709,6 +709,7 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat 
		       spdk_idxd_req_cb cb_fn, void *cb_arg)

{

	struct idxd_hw_desc *desc;

	struct idxd_comp *comp;

	uint64_t desc_addr;

	int i, rc;
@@ -725,10 +726,9 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat 
	}



	/* Common prep. */

	desc = _idxd_prep_command(chan, cb_fn, cb_arg, batch);

	if (desc == NULL) {

		SPDK_DEBUGLOG(idxd, "Busy, can't submit batch %p\n", batch);

		return -EBUSY;

	rc = _idxd_prep_command(chan, cb_fn, cb_arg, &desc, &comp);

	if (rc) {

		return rc;

	}



	rc = _vtophys(batch->user_desc, &desc_addr, batch->remaining * sizeof(struct idxd_hw_desc));
@@ -740,6 +740,7 @@ spdk_idxd_batch_submit(struct spdk_idxd_io_channel *chan, struct idxd_batch *bat 
	desc->opcode = IDXD_OPCODE_BATCH;

	desc->desc_list_addr = desc_addr;

	desc->desc_count = batch->remaining = batch->index;

	comp->batch = batch;

	assert(batch->index <= DESC_PER_BATCH);



	/* Add the batch elements completion contexts to the outstanding list to be polled. */
@@ -1042,8 +1043,8 @@ spdk_idxd_process_events(struct spdk_idxd_io_channel *chan) 
				SPDK_DEBUGLOG(idxd, "Complete batch %p\n", comp_ctx->batch);

				break;

			case IDXD_OPCODE_CRC32C_GEN:

				*(uint32_t *)comp_ctx->desc->dst_addr = comp_ctx->hw.crc32c_val;

				*(uint32_t *)comp_ctx->desc->dst_addr ^= ~0;

				*(uint32_t *)comp_ctx->crc_dst = comp_ctx->hw.crc32c_val;

				*(uint32_t *)comp_ctx->crc_dst ^= ~0;

				break;

			case IDXD_OPCODE_COMPARE:

				if (status == 0) {

lib/idxd/idxd.h

+2 −1
Original line number Diff line number Diff line
@@ -164,9 +164,10 @@ struct idxd_comp { 
	void				*cb_arg;

	spdk_idxd_req_cb		cb_fn;

	struct idxd_batch		*batch;

	bool				batch_op;

	struct idxd_hw_desc		*desc;

	uint32_t			*crc_dst;

	uint32_t			index;

	bool				batch_op;

	char				pad[3];

	TAILQ_ENTRY(idxd_comp)		link;

};