Commit 59ed0fd8 authored by Tomasz Zawadzki's avatar Tomasz Zawadzki
Browse files

lib/idxd: match batch size to max reported from device 



Rather than limiting the batch size to DESC_PER_BATCH,
use the maximum batch size value presented from the device.
To accommodate increased the allocated descriptors for a batch.

Prior to this patch full batch was flushed on task submission,
meanwhile any built up batch was submitted in spdk_idxd_process_events().

Rate of flushing is now defined by IDXD_MIN_BATCH_FLUSH,
while still allowing for larger batches to be created within single
task submission.
If IDXD_MIN_BATCH_FLUSH is modified to very high value, batches
would still be submitted in spdk_idxd_process_events().

For now the IDXD_MIN_BATCH_FLUSH matches prior DESC_PER_BATCH limit,
making this parameter configurable might be explored.

Signed-off-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Change-Id: Ie32648731260a542c858a8dbc91314d6b353e2ce
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21243


Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarSlawomir Ptak <slawomir.ptak@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Community-CI: Mellanox Build Bot
parent 84c03605

lib/idxd/idxd.c

+12 −7
Original line number Diff line number Diff line
@@ -22,6 +22,9 @@ 
/* The max number of completions processed per poll */

#define IDXD_MAX_COMPLETIONS      128



/* The minimum number of entries in batch per flush */

#define IDXD_MIN_BATCH_FLUSH      32



static STAILQ_HEAD(, spdk_idxd_impl) g_idxd_impls = STAILQ_HEAD_INITIALIZER(g_idxd_impls);

static struct spdk_idxd_impl *g_idxd_impl;
@@ -155,7 +158,8 @@ _dsa_alloc_batches(struct spdk_idxd_io_channel *chan, int num_descriptors) 
	}

	batch = chan->batch_base;

	for (i = 0 ; i < num_batches ; i++) {

		batch->user_desc = desc = spdk_zmalloc(DESC_PER_BATCH * sizeof(struct idxd_hw_desc),

		batch->size = chan->idxd->batch_size;

		batch->user_desc = desc = spdk_zmalloc(batch->size * sizeof(struct idxd_hw_desc),

						       0x40, NULL,

						       SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);

		if (batch->user_desc == NULL) {
@@ -164,13 +168,13 @@ _dsa_alloc_batches(struct spdk_idxd_io_channel *chan, int num_descriptors) 
		}



		rc = _vtophys(chan, batch->user_desc, &batch->user_desc_addr,

			      DESC_PER_BATCH * sizeof(struct idxd_hw_desc));

			      batch->size * sizeof(struct idxd_hw_desc));

		if (rc) {

			SPDK_ERRLOG("Failed to translate batch descriptor memory\n");

			goto error_user;

		}



		batch->user_ops = op = spdk_zmalloc(DESC_PER_BATCH * sizeof(struct idxd_ops),

		batch->user_ops = op = spdk_zmalloc(batch->size * sizeof(struct idxd_ops),

						    0x40, NULL,

						    SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);

		if (batch->user_ops == NULL) {
@@ -178,7 +182,7 @@ _dsa_alloc_batches(struct spdk_idxd_io_channel *chan, int num_descriptors) 
			goto error_user;

		}



		for (j = 0; j < DESC_PER_BATCH; j++) {

		for (j = 0; j < batch->size; j++) {

			rc = _vtophys(chan, &op->hw, &desc->completion_addr, sizeof(struct dsa_hw_comp_record));

			if (rc) {

				SPDK_ERRLOG("Failed to translate batch entry completion memory\n");
@@ -442,7 +446,7 @@ _idxd_prep_batch_cmd(struct spdk_idxd_io_channel *chan, spdk_idxd_req_cb cb_fn, 
	batch = chan->batch;



	assert(batch != NULL);

	if (batch->index == DESC_PER_BATCH) {

	if (batch->index == batch->size) {

		return -EBUSY;

	}
@@ -573,7 +577,7 @@ idxd_batch_submit(struct spdk_idxd_io_channel *chan, 
		desc->opcode = IDXD_OPCODE_BATCH;

		desc->desc_list_addr = batch->user_desc_addr;

		desc->desc_count = batch->index;

		assert(batch->index <= DESC_PER_BATCH);

		assert(batch->index <= batch->size);



		/* Add the batch elements completion contexts to the outstanding list to be polled. */

		for (i = 0 ; i < batch->index; i++) {
@@ -611,9 +615,10 @@ _idxd_setup_batch(struct spdk_idxd_io_channel *chan) 
static int

_idxd_flush_batch(struct spdk_idxd_io_channel *chan)

{

	struct idxd_batch *batch = chan->batch;

	int rc;



	if (chan->batch != NULL && chan->batch->index >= DESC_PER_BATCH) {

	if (batch != NULL && batch->index >= IDXD_MIN_BATCH_FLUSH) {

		/* Close out the full batch */

		rc = idxd_batch_submit(chan, NULL, NULL);

		if (rc) {

lib/idxd/idxd_internal.h

+2 −4
Original line number Diff line number Diff line
@@ -33,10 +33,6 @@ static inline void movdir64b(void *dst, const void *src) 


/* TODO: consider setting the max per batch limit via RPC. */



/* The following sets up a max desc count per batch of 32 */

#define LOG2_WQ_MAX_BATCH	5  /* 2^5 = 32 */

#define DESC_PER_BATCH		(1 << LOG2_WQ_MAX_BATCH)



#define LOG2_WQ_MAX_XFER	30 /* 2^30 = 1073741824 */

#define WQ_PRIORITY_1		1

#define IDXD_MAX_QUEUES		64
@@ -55,6 +51,7 @@ struct idxd_batch { 
	uint64_t			user_desc_addr;

	uint16_t			index;

	uint16_t			refcnt;

	uint16_t			size;

	struct spdk_idxd_io_channel	*chan;

	TAILQ_ENTRY(idxd_batch)		link;

};
@@ -139,6 +136,7 @@ struct spdk_idxd_device { 
	uint32_t			num_channels;

	uint32_t			total_wq_size;

	uint32_t			chan_per_device;

	uint16_t			batch_size;

	pthread_mutex_t			num_channels_lock;

	bool				pasid_enabled;

	enum idxd_dev			type;

lib/idxd/idxd_user.c

+3 −1
Original line number Diff line number Diff line
@@ -222,11 +222,13 @@ idxd_wq_config(struct spdk_user_idxd_device *user_idxd) 


	wqcfg->wq_size = wqcap.total_wq_size;

	wqcfg->mode = WQ_MODE_DEDICATED;

	wqcfg->max_batch_shift = LOG2_WQ_MAX_BATCH;

	wqcfg->max_batch_shift = user_idxd->registers->gencap.max_batch_shift;

	wqcfg->max_xfer_shift = LOG2_WQ_MAX_XFER;

	wqcfg->wq_state = WQ_ENABLED;

	wqcfg->priority = WQ_PRIORITY_1;



	idxd->batch_size = (1 << wqcfg->max_batch_shift);



	for (i = 0; i < SPDK_COUNTOF(wqcfg->raw); i++) {

		spdk_mmio_write_4(&wqcfg->raw[i], wqcfg->raw[i]);

	}

test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c

+1 −0
Original line number Diff line number Diff line
@@ -72,6 +72,7 @@ spdk_pci_device_cfg_write32(struct spdk_pci_device *dev, uint32_t value, 


#define WQ_CFG_OFFSET (0x800 / IDXD_TABLE_OFFSET_MULT)

#define TOTAL_WQE_SIZE 0x40

#define LOG2_WQ_MAX_BATCH 5

static int

test_idxd_wq_config(void)

{