Commit c501d2b3 authored by paul luse's avatar paul luse Committed by Jim Harris
Browse files

idxd: fix issue w/multiple WQ config 



Found via inspection during spec review of latest HW. We were using the
wrong stride for the WQCFG regsiter when configuring but it just so
happened to be the right value for the current DSA version.  We were
mixing up the size of the WQCFG register with the stride value used to
configure the next WQCFG regsiter as they are not contiguous in HW, we
need to read another capabilities bit to determine the address of the
next wqcfg to configure..

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: I14d1ff95e0131fd30121aa955bfbc8c8fb3fc512
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/10968


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 026f0031

include/spdk/idxd_spec.h

+1 −0
Original line number Diff line number Diff line
@@ -52,6 +52,7 @@ extern "C" { 
#define WQ_TOTAL_PORTAL_SIZE		(PORTAL_SIZE * 4)

#define PORTAL_STRIDE			0x40

#define PORTAL_MASK			(PORTAL_SIZE - 1)

#define WQCFG_SHIFT			5



#define CFG_ENGINE_OFFSET		0x20

#define CFG_FLAG_OFFSET			0x28

lib/idxd/idxd.h

+0 −1
Original line number Diff line number Diff line
@@ -66,7 +66,6 @@ static inline void movdir64b(void *dst, const void *src) 
#define DESC_PER_BATCH		(1 << LOG2_WQ_MAX_BATCH)



#define LOG2_WQ_MAX_XFER	30 /* 2^30 = 1073741824 */

#define WQCFG_NUM_DWORDS	8

#define WQ_PRIORITY_1		1

#define IDXD_MAX_QUEUES		64

lib/idxd/idxd_user.c

+5 −4
Original line number Diff line number Diff line
@@ -261,10 +261,11 @@ idxd_group_config(struct spdk_idxd_device *idxd) 
static int

idxd_wq_config(struct spdk_user_idxd_device *user_idxd)

{

	int i, j;

	uint32_t i, j;

	struct idxd_wq *queue;

	struct spdk_idxd_device *idxd = &user_idxd->idxd;

	u_int32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;

	uint32_t wq_size = user_idxd->registers.wqcap.total_wq_size / g_user_dev_cfg.total_wqs;

	uint32_t wqcap_size = 1 << (WQCFG_SHIFT + user_idxd->registers.wqcap.wqcfg_size);



	SPDK_DEBUGLOG(idxd, "Total ring slots available space 0x%x, so per work queue is 0x%x\n",

		      user_idxd->registers.wqcap.total_wq_size, wq_size);
@@ -303,8 +304,8 @@ idxd_wq_config(struct spdk_user_idxd_device *user_idxd) 
	 */

	for (i = 0 ; i < user_idxd->registers.wqcap.num_wqs; i++) {

		queue = &idxd->queues[i];

		for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {

			_idxd_write_4(idxd, user_idxd->wqcfg_offset + i * 32 + j * 4,

		for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) {

			_idxd_write_4(idxd, user_idxd->wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t),

				      queue->wqcfg.raw[j]);

		}

	}

test/unit/lib/idxd/idxd_user.c/idxd_user_ut.c

+6 −6
Original line number Diff line number Diff line
@@ -108,8 +108,9 @@ test_idxd_wq_config(void) 
	struct spdk_idxd_device *idxd = &user_idxd.idxd;

	union idxd_wqcfg wqcfg = {};

	uint32_t expected[8] = {0x40, 0, 0x11, 0xbe, 0, 0, 0x40000000, 0};

	uint32_t wq_size;

	int rc, i, j;

	uint32_t wq_size, i, j;

	uint32_t wqcap_size = 32;

	int rc;



	user_idxd.reg_base = calloc(1, FAKE_REG_SIZE);

	SPDK_CU_ASSERT_FATAL(user_idxd.reg_base != NULL);
@@ -139,10 +140,9 @@ test_idxd_wq_config(void) 
	}



	for (i = 0 ; i < user_idxd.registers.wqcap.num_wqs; i++) {

		for (j = 0 ; j < WQCFG_NUM_DWORDS; j++) {

			wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base + user_idxd.wqcfg_offset + i * 32 +

							j *

							4));

		for (j = 0 ; j < (sizeof(union idxd_wqcfg) / sizeof(uint32_t)); j++) {

			wqcfg.raw[j] = spdk_mmio_read_4((uint32_t *)(user_idxd.reg_base +

							user_idxd.wqcfg_offset + i * wqcap_size + j * sizeof(uint32_t)));

			CU_ASSERT(wqcfg.raw[j] == expected[j]);

		}

	}