Commit ecaa8e10 authored by paul luse's avatar paul luse Committed by Tomasz Zawadzki
Browse files

lib/idxd: prepare some plumbing for adding IAA 



Misc internal IDXD changes needed to support the upcoming addition
of IAA.

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: Idb180088af545b174ed33a4f8ee113e58640477f
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/12764


Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 9c55555c

examples/idxd/perf/perf.c

+9 −2
Original line number Diff line number Diff line
@@ -166,13 +166,20 @@ attach_cb(void *cb_ctx, struct spdk_idxd_device *idxd) 
	g_num_devices++;

}



static bool

probe_cb(void *cb_ctx, struct spdk_pci_device *dev)

{

	/* this tool will gladly claim all types of IDXD devices. */

	return true;

}



static int

idxd_init(void)

{

	spdk_idxd_set_config(g_idxd_kernel_mode);



	if (spdk_idxd_probe(NULL, attach_cb) != 0) {

		fprintf(stderr, "idxd_probe() failed\n");

	if (spdk_idxd_probe(NULL, attach_cb, probe_cb) != 0) {

		fprintf(stderr, "spdk_idxd_probe() failed\n");

		return 1;

	}

include/spdk/idxd.h

+12 −1
Original line number Diff line number Diff line
@@ -104,6 +104,15 @@ typedef void (*spdk_idxd_req_cb)(void *arg, int status); 
 */

typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_idxd_device *idxd);



/**

 * Callback for spdk_idxd_probe() to report a device that has been found.

 *

 * \param cb_ctx User-specified opaque value corresponding to cb_ctx from spdk_idxd_probe().

 * \param dev PCI device that is in question.

 * \return true if the caller wants the device, false if not..

 */

typedef bool (*spdk_idxd_probe_cb)(void *cb_ctx, struct spdk_pci_device *dev);



/**

 * Enumerate the IDXD devices attached to the system and attach the userspace

 * IDXD driver to them if desired.
@@ -116,12 +125,14 @@ typedef void (*spdk_idxd_attach_cb)(void *cb_ctx, struct spdk_idxd_device *idxd) 
 *

 * \param cb_ctx Opaque value which will be passed back in cb_ctx parameter of

 * the callbacks.

 * \param probe_cb callback to determine if the device being probe should be attached.

 * \param attach_cb will be called for devices for which probe_cb returned true

 * once the IDXD controller has been attached to the userspace driver.

 *

 * \return 0 on success, -1 on failure.

 */

int spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb);

int spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb,

		    spdk_idxd_probe_cb probe_cb);



/**

 * Detach specified device returned by spdk_idxd_probe() from the IDXD driver.

include/spdk/idxd_spec.h

+56 −1
Original line number Diff line number Diff line
@@ -100,6 +100,32 @@ enum dsa_completion_status { 
	DSA_COMP_TRANSLATION_FAIL	= 34,

};



enum iaa_completion_status {

	IAA_COMP_NONE			= 0,

	IAA_COMP_SUCCESS		= 1,

	IAA_COMP_PAGE_FAULT_IR		= 4,

	IAA_COMP_OUTBUF_OVERFLOW	= 5,

	IAA_COMP_BAD_OPCODE		= 16,

	IAA_COMP_INVALID_FLAGS		= 17,

	IAA_COMP_NOZERO_RESERVE		= 18,

	IAA_COMP_INVALID_SIZE		= 19,

	IAA_COMP_OVERLAP_BUFFERS	= 22,

	IAA_COMP_INT_HANDLE_INVAL	= 25,

	IAA_COMP_CRA_XLAT		= 32,

	IAA_COMP_CRA_ALIGN		= 33,

	IAA_COMP_ADDR_ALIGN		= 34,

	IAA_COMP_PRIV_BAD		= 35,

	IAA_COMP_TRAFFIC_CLASS_CONF	= 36,

	IAA_COMP_PFAULT_RDBA		= 37,

	IAA_COMP_HW_ERR1		= 38,

	IAA_COMP_TRANSLATION_FAIL	= 39,

	IAA_COMP_PRS_TIMEOUT		= 40,

	IAA_COMP_WATCHDOG		= 41,

	IAA_COMP_INVALID_COMP_FLAG	= 48,

	IAA_COMP_INVALID_FILTER_FLAG	= 49,

	IAA_COMP_INVALID_NUM_ELEMS	= 50,

};



enum idxd_wq_state {

	WQ_DISABLED	= 0,

	WQ_ENABLED	= 1,
@@ -188,6 +214,7 @@ struct idxd_hw_desc { 
	uint64_t	completion_addr;

	union {

		uint64_t	src_addr;

		uint64_t	src1_addr;

		uint64_t	readback_addr;

		uint64_t	pattern;

		uint64_t	desc_list_addr;
@@ -199,12 +226,24 @@ struct idxd_hw_desc { 
		uint64_t	comp_pattern;

	};

	union {

		uint32_t	src1_size;

		uint32_t	xfer_size;

		uint32_t	desc_count;

	};

	uint16_t	int_handle;

	union {

		uint16_t	rsvd1;

		uint16_t	compr_flags;

		uint16_t	decompr_flags;

	};

	union {

		struct {

			uint64_t	src2_addr;

			uint32_t	max_dst_size;

			uint32_t	src2_size;

			uint32_t	filter_flags;

			uint32_t	num_inputs;

		} iaa;

		uint8_t		expected_res;

		struct {

			uint64_t	addr;
@@ -288,6 +327,22 @@ struct dsa_hw_comp_record { 
};

SPDK_STATIC_ASSERT(sizeof(struct dsa_hw_comp_record) == 32, "size mismatch");



struct iaa_hw_comp_record {

	volatile uint8_t	status;

	uint8_t			error_code;

	uint16_t		rsvd;

	uint32_t		bytes_completed;

	uint64_t		fault_addr;

	uint32_t		invalid_flags;

	uint32_t		rsvd2;

	uint32_t		output_size;

	uint8_t			output_bits;

	uint8_t			rsvd3;

	uint16_t		rsvd4;

	uint64_t		rsvd5[4];

};

SPDK_STATIC_ASSERT(sizeof(struct iaa_hw_comp_record) == 64, "size mismatch");



union idxd_gencap_register {

	struct {

		uint64_t block_on_fault: 1;

lib/idxd/idxd.c

+3 −2
Original line number Diff line number Diff line
@@ -348,14 +348,15 @@ idxd_device_destruct(struct spdk_idxd_device *idxd) 
}



int

spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb)

spdk_idxd_probe(void *cb_ctx, spdk_idxd_attach_cb attach_cb,

		spdk_idxd_probe_cb probe_cb)

{

	if (g_idxd_impl == NULL) {

		SPDK_ERRLOG("No idxd impl is selected\n");

		return -1;

	}



	return g_idxd_impl->probe(cb_ctx, attach_cb);

	return g_idxd_impl->probe(cb_ctx, attach_cb, probe_cb);

}



void

lib/idxd/idxd.h

+13 −3
Original line number Diff line number Diff line
@@ -69,6 +69,11 @@ static inline void movdir64b(void *dst, const void *src) 
#define WQ_PRIORITY_1		1

#define IDXD_MAX_QUEUES		64



enum idxd_dev {

	IDXD_DEV_TYPE_DSA	= 0,

	IDXD_DEV_TYPE_IAA	= 1,

};



/* Each pre-allocated batch structure goes on a per channel list and

 * contains the memory for both user descriptors.

 */
@@ -124,7 +129,10 @@ struct pci_dev_id { 
 * size and must be 32 byte aligned.

 */

struct idxd_ops {

	union {

		struct dsa_hw_comp_record	hw;

		struct iaa_hw_comp_record	iaa_hw;

	};

	void				*cb_arg;

	spdk_idxd_req_cb		cb_fn;

	struct idxd_batch		*batch;
@@ -134,11 +142,12 @@ struct idxd_ops { 
	uint32_t			count;

	STAILQ_ENTRY(idxd_ops)		link;

};

SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 96, "size mismatch");

SPDK_STATIC_ASSERT(sizeof(struct idxd_ops) == 128, "size mismatch");



struct spdk_idxd_impl {

	const char *name;

	int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb);

	int (*probe)(void *cb_ctx, spdk_idxd_attach_cb attach_cb,

		     spdk_idxd_probe_cb probe_cb);

	void (*destruct)(struct spdk_idxd_device *idxd);

	void (*dump_sw_error)(struct spdk_idxd_device *idxd, void *portal);

	char *(*portal_get_addr)(struct spdk_idxd_device *idxd);
@@ -154,6 +163,7 @@ struct spdk_idxd_device { 
	uint32_t			total_wq_size;

	uint32_t			chan_per_device;

	pthread_mutex_t			num_channels_lock;

	enum idxd_dev			type;

};



void idxd_impl_register(struct spdk_idxd_impl *impl);