Commit 8209c8cf authored by Seth Howell's avatar Seth Howell Committed by Tomasz Zawadzki
Browse files

lib/nvmf: rdma qpairs now store device, not port. 



The only reason the qpair ever uses the port is to get to
the device attributes so skip the middle man.

Signed-off-by: default avatarSeth Howell <seth.howell@intel.com>
Change-Id: Ib14a97ceaa0c49176027d6c35c5cb2787a845cc1
Reviewed-on: https://review.gerrithub.io/c/spdk/spdk/+/478961


Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Community-CI: SPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarShuhei Matsumoto <shuhei.matsumoto.xt@hitachi.com>
parent f5916679

lib/nvmf/rdma.c

+6 −6
Original line number Diff line number Diff line
@@ -342,7 +342,7 @@ struct spdk_nvmf_rdma_ibv_event_ctx { 
struct spdk_nvmf_rdma_qpair {

	struct spdk_nvmf_qpair			qpair;



	struct spdk_nvmf_rdma_port		*port;

	struct spdk_nvmf_rdma_device		*device;

	struct spdk_nvmf_rdma_poller		*poller;



	struct rdma_cm_id			*cm_id;
@@ -1024,7 +1024,7 @@ spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
	struct ibv_qp_init_attr			ibv_init_attr;



	rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);

	device = rqpair->port->device;

	device = rqpair->device;



	memset(&ibv_init_attr, 0, sizeof(struct ibv_qp_init_attr));

	ibv_init_attr.qp_context	= rqpair;
@@ -1049,7 +1049,7 @@ spdk_nvmf_rdma_qpair_initialize(struct spdk_nvmf_qpair *qpair) 
		goto error;

	}



	rc = rdma_create_qp(rqpair->cm_id, rqpair->port->device->pd, &ibv_init_attr);

	rc = rdma_create_qp(rqpair->cm_id, device->pd, &ibv_init_attr);

	if (rc) {

		SPDK_ERRLOG("rdma_create_qp failed: errno %d: %s\n", errno, spdk_strerror(errno));

		goto error;
@@ -1350,7 +1350,7 @@ nvmf_rdma_connect(struct spdk_nvmf_transport *transport, struct rdma_cm_event *e 
		return -1;

	}



	rqpair->port = port;

	rqpair->device = port->device;

	rqpair->max_queue_depth = max_queue_depth;

	rqpair->max_read_depth = max_read_depth;

	rqpair->cm_id = event->id;
@@ -2011,7 +2011,7 @@ spdk_nvmf_rdma_request_process(struct spdk_nvmf_rdma_transport *rtransport, 
	uint32_t			num_blocks;



	rqpair = SPDK_CONTAINEROF(rdma_req->req.qpair, struct spdk_nvmf_rdma_qpair, qpair);

	device = rqpair->port->device;

	device = rqpair->device;

	rgroup = rqpair->poller->group;



	assert(rdma_req->state != RDMA_REQUEST_STATE_FREE);
@@ -3456,7 +3456,7 @@ spdk_nvmf_rdma_poll_group_add(struct spdk_nvmf_transport_poll_group *group, 
	rgroup = SPDK_CONTAINEROF(group, struct spdk_nvmf_rdma_poll_group, group);

	rqpair = SPDK_CONTAINEROF(qpair, struct spdk_nvmf_rdma_qpair, qpair);



	device = rqpair->port->device;

	device = rqpair->device;



	TAILQ_FOREACH(poller, &rgroup->pollers, link) {

		if (poller->device == device) {

test/unit/lib/nvmf/rdma.c/rdma_ut.c

+6 −8
Original line number Diff line number Diff line
@@ -535,14 +535,14 @@ free_req(struct spdk_nvmf_rdma_request *rdma_req) 
static void

qpair_reset(struct spdk_nvmf_rdma_qpair *rqpair,

	    struct spdk_nvmf_rdma_poller *poller,

	    struct spdk_nvmf_rdma_port *port,

	    struct spdk_nvmf_rdma_device *device,

	    struct spdk_nvmf_rdma_resources *resources)

{

	memset(rqpair, 0, sizeof(*rqpair));

	STAILQ_INIT(&rqpair->pending_rdma_write_queue);

	STAILQ_INIT(&rqpair->pending_rdma_read_queue);

	rqpair->poller = poller;

	rqpair->port = port;

	rqpair->device = device;

	rqpair->resources = resources;

	rqpair->qpair.qid = 1;

	rqpair->ibv_state = IBV_QPS_RTS;
@@ -569,7 +569,6 @@ test_spdk_nvmf_rdma_request_process(void) 
	struct spdk_nvmf_rdma_transport rtransport = {};

	struct spdk_nvmf_rdma_poll_group group = {};

	struct spdk_nvmf_rdma_poller poller = {};

	struct spdk_nvmf_rdma_port port = {};

	struct spdk_nvmf_rdma_device device = {};

	struct spdk_nvmf_rdma_resources resources = {};

	struct spdk_nvmf_rdma_qpair rqpair = {};
@@ -581,9 +580,8 @@ test_spdk_nvmf_rdma_request_process(void) 
	STAILQ_INIT(&group.group.pending_buf_queue);

	group.group.buf_cache_size = 0;

	group.group.buf_cache_count = 0;

	port.device = &device;

	poller_reset(&poller, &group);

	qpair_reset(&rqpair, &poller, &port, &resources);

	qpair_reset(&rqpair, &poller, &device, &resources);



	rtransport.transport.opts = g_rdma_ut_transport_opts;

	rtransport.transport.data_buf_pool = spdk_mempool_create("test_data_pool", 16, 128, 0, 0);
@@ -624,7 +622,7 @@ test_spdk_nvmf_rdma_request_process(void) 
	free_recv(rdma_recv);

	free_req(rdma_req);

	poller_reset(&poller, &group);

	qpair_reset(&rqpair, &poller, &port, &resources);

	qpair_reset(&rqpair, &poller, &device, &resources);



	/* Test 2: single SGL WRITE request */

	rdma_recv = create_recv(&rqpair, SPDK_NVME_OPC_WRITE);
@@ -663,7 +661,7 @@ test_spdk_nvmf_rdma_request_process(void) 
	free_recv(rdma_recv);

	free_req(rdma_req);

	poller_reset(&poller, &group);

	qpair_reset(&rqpair, &poller, &port, &resources);

	qpair_reset(&rqpair, &poller, &device, &resources);



	/* Test 3: WRITE+WRITE ibv_send batching */

	{
@@ -736,7 +734,7 @@ test_spdk_nvmf_rdma_request_process(void) 
		free_recv(recv2);

		free_req(req2);

		poller_reset(&poller, &group);

		qpair_reset(&rqpair, &poller, &port, &resources);

		qpair_reset(&rqpair, &poller, &device, &resources);

	}



	spdk_mempool_free(rtransport.transport.data_buf_pool);